feat(provenance): govern source snapshots and data inputs
This commit is contained in:
@@ -26,6 +26,10 @@ from app.db.session import SessionLocal # noqa: E402
|
||||
from app.models import Dataset # noqa: E402
|
||||
|
||||
from normalize_belgium_building_labels import normalize # noqa: E402
|
||||
from training_dataset_eligibility import ( # noqa: E402
|
||||
TRAINING_ELIGIBILITY_POLICY_VERSION,
|
||||
training_pair_evidence,
|
||||
)
|
||||
|
||||
REGION_SOURCES = {
|
||||
"flanders": ({"digitaal_vlaanderen_orthophoto"}, "grb"),
|
||||
@@ -52,7 +56,13 @@ def _dataset_path(dataset: Dataset) -> Path:
|
||||
return path
|
||||
|
||||
|
||||
def _validate_pair(sample: dict[str, Any], raster: Dataset, reference: Dataset) -> tuple[str, str]:
|
||||
def _validate_pair(
|
||||
sample: dict[str, Any],
|
||||
raster: Dataset,
|
||||
reference: Dataset,
|
||||
*,
|
||||
fixture_mode: bool,
|
||||
) -> tuple[str, str, dict[str, Any]]:
|
||||
region = str(sample.get("region") or "").lower()
|
||||
if region not in REGION_SOURCES:
|
||||
raise SystemExit(f"Unsupported region for {sample.get('sample_slug')}: {region}")
|
||||
@@ -66,7 +76,23 @@ def _validate_pair(sample: dict[str, Any], raster: Dataset, reference: Dataset)
|
||||
raise SystemExit(f"Unsupported split for {sample['sample_slug']}: {split}")
|
||||
if raster.status != "ready" or reference.status != "ready":
|
||||
raise SystemExit(f"Dataset pair is not ready for {sample['sample_slug']}")
|
||||
return region, expected_reference
|
||||
eligibility = training_pair_evidence(
|
||||
raster=raster,
|
||||
reference=reference,
|
||||
fixture_mode=fixture_mode,
|
||||
)
|
||||
if not eligibility["eligible"]:
|
||||
reasons = sorted(
|
||||
{
|
||||
reason
|
||||
for role in ("raster", "reference")
|
||||
for reason in eligibility[role]["reasons"]
|
||||
}
|
||||
)
|
||||
raise SystemExit(
|
||||
f"Dataset pair is not eligible for training for {sample['sample_slug']}: {', '.join(reasons)}"
|
||||
)
|
||||
return region, expected_reference, eligibility
|
||||
|
||||
|
||||
def audit_spatial_leakage(samples: list[dict[str, Any]], buffer_m: float = 64.0) -> dict[str, Any]:
|
||||
@@ -104,6 +130,14 @@ def main() -> int:
|
||||
parser.add_argument("--min-label-px", type=float, default=3.0)
|
||||
parser.add_argument("--merge-touching-roofs", action="store_true")
|
||||
parser.add_argument("--freeze", action="store_true")
|
||||
parser.add_argument(
|
||||
"--fixture-mode",
|
||||
action="store_true",
|
||||
help=(
|
||||
"Allow only explicitly marked fixture datasets with legacy provenance. "
|
||||
"Never use this mode for an operational corpus."
|
||||
),
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
spec = json.loads(args.spec.read_text(encoding="utf-8-sig"))
|
||||
@@ -129,7 +163,12 @@ def main() -> int:
|
||||
reference = db.get(Dataset, UUID(str(sample["reference_dataset_id"])))
|
||||
if raster is None or reference is None:
|
||||
raise SystemExit(f"Persisted Dataset pair not found for {slug}")
|
||||
region, reference_source = _validate_pair(sample, raster, reference)
|
||||
region, reference_source, eligibility = _validate_pair(
|
||||
sample,
|
||||
raster,
|
||||
reference,
|
||||
fixture_mode=args.fixture_mode,
|
||||
)
|
||||
raster_source = _dataset_path(raster)
|
||||
reference_source_path = _dataset_path(reference)
|
||||
sample_dir = pairs_dir / slug
|
||||
@@ -173,14 +212,20 @@ def main() -> int:
|
||||
"raster_sha256": sha256(raster_target),
|
||||
"reference_sha256": sha256(normalized_target),
|
||||
"label_audit_sha256": sha256(audit_target),
|
||||
"training_eligibility": eligibility,
|
||||
"bbox_epsg4326": (raster.source_metadata or {}).get("bbox_epsg4326")
|
||||
or sample.get("bbox_epsg4326"),
|
||||
}
|
||||
)
|
||||
manifest = {
|
||||
"schema_version": 1,
|
||||
"schema_version": 2,
|
||||
"dataset_version": args.version,
|
||||
"immutable": bool(args.freeze),
|
||||
"training_eligibility": {
|
||||
"policy_version": TRAINING_ELIGIBILITY_POLICY_VERSION,
|
||||
"status": "eligible",
|
||||
"fixture_mode": bool(args.fixture_mode),
|
||||
},
|
||||
"samples": manifest_samples,
|
||||
}
|
||||
manifest_path = output_dir / "operator_samples_manifest.json"
|
||||
@@ -192,10 +237,13 @@ def main() -> int:
|
||||
if leakage_audit["status"] != "ok":
|
||||
raise SystemExit("Spatial split leakage audit failed")
|
||||
freeze = {
|
||||
"schema_version": 2,
|
||||
"dataset_version": args.version,
|
||||
"manifest_sha256": sha256(manifest_path),
|
||||
"sample_count": len(manifest_samples),
|
||||
"immutable": bool(args.freeze),
|
||||
"training_eligibility_policy": TRAINING_ELIGIBILITY_POLICY_VERSION,
|
||||
"fixture_mode": bool(args.fixture_mode),
|
||||
}
|
||||
(output_dir / "corpus-freeze.json").write_text(json.dumps(freeze, indent=2), encoding="utf-8")
|
||||
print(json.dumps(freeze))
|
||||
|
||||
@@ -4,13 +4,49 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import sys
|
||||
from collections import Counter
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
SCRIPT_DIR = Path(__file__).resolve().parent
|
||||
if str(SCRIPT_DIR) not in sys.path:
|
||||
sys.path.insert(0, str(SCRIPT_DIR))
|
||||
|
||||
from training_dataset_eligibility import frozen_manifest_training_eligibility_failures # noqa: E402
|
||||
|
||||
REQUIRED_SPLITS = ("train", "val", "calibration", "test", "background-test")
|
||||
REGIONS = ("flanders", "wallonia", "brussels")
|
||||
SHA256_LENGTH = 64
|
||||
|
||||
|
||||
def sha256(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as stream:
|
||||
for chunk in iter(lambda: stream.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def valid_review_timestamp(value: Any) -> bool:
|
||||
if not isinstance(value, str) or not value.strip():
|
||||
return False
|
||||
try:
|
||||
parsed = datetime.fromisoformat(value.strip().replace("Z", "+00:00"))
|
||||
except ValueError:
|
||||
return False
|
||||
return parsed.tzinfo is not None
|
||||
|
||||
|
||||
def valid_sha256(value: Any) -> bool:
|
||||
return (
|
||||
isinstance(value, str)
|
||||
and len(value) == SHA256_LENGTH
|
||||
and all(character in "0123456789abcdefABCDEF" for character in value)
|
||||
)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
@@ -19,8 +55,15 @@ def main() -> int:
|
||||
parser.add_argument("--output-dir", type=Path, required=True)
|
||||
parser.add_argument("--review-decisions", type=Path)
|
||||
args = parser.parse_args()
|
||||
manifest = json.loads((args.corpus_dir / "operator_samples_manifest.json").read_text(encoding="utf-8"))
|
||||
manifest_path = args.corpus_dir / "operator_samples_manifest.json"
|
||||
manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
|
||||
leakage = json.loads((args.corpus_dir / "spatial-leakage-audit.json").read_text(encoding="utf-8"))
|
||||
manifest_policy = manifest.get("training_eligibility")
|
||||
fixture_mode = bool(manifest_policy.get("fixture_mode")) if isinstance(manifest_policy, dict) else False
|
||||
eligibility_failures = frozen_manifest_training_eligibility_failures(
|
||||
manifest_path,
|
||||
fixture_mode=fixture_mode,
|
||||
)
|
||||
samples = manifest["samples"]
|
||||
split_counts = Counter((sample["region"], sample["split"]) for sample in samples)
|
||||
decision_counts: Counter[str] = Counter()
|
||||
@@ -28,7 +71,7 @@ def main() -> int:
|
||||
total_input = 0
|
||||
total_accepted = 0
|
||||
temporal_unknown = 0
|
||||
failures: list[str] = []
|
||||
failures: list[str] = list(eligibility_failures)
|
||||
for region in REGIONS:
|
||||
for split in REQUIRED_SPLITS:
|
||||
minimum = 4 if split == "train" else 2
|
||||
@@ -62,8 +105,12 @@ def main() -> int:
|
||||
failures.append("spatial leakage audit failed")
|
||||
reviewed = 0
|
||||
review_complete = False
|
||||
review_evidence_failures: list[str] = []
|
||||
accepted_review_evidence: dict[str, dict[str, Any]] = {}
|
||||
review_decisions_path: Path | None = None
|
||||
if args.review_decisions and args.review_decisions.is_file():
|
||||
decisions = json.loads(args.review_decisions.read_text(encoding="utf-8"))
|
||||
review_decisions_path = args.review_decisions.resolve(strict=False)
|
||||
decisions = json.loads(review_decisions_path.read_text(encoding="utf-8"))
|
||||
by_slug = {item["sample_slug"]: item for item in decisions.get("decisions", [])}
|
||||
for item in review_queue:
|
||||
decision = by_slug.get(item["sample_slug"])
|
||||
@@ -71,13 +118,70 @@ def main() -> int:
|
||||
item["decision"] = decision.get("decision")
|
||||
item["reviewer"] = decision.get("reviewer")
|
||||
item["notes"] = decision.get("notes")
|
||||
if item["decision"] in {"accepted", "rejected"} and item.get("reviewer"):
|
||||
item["reviewed_at"] = decision.get("reviewed_at")
|
||||
item["reviewed_artifact_path"] = decision.get("reviewed_artifact_path")
|
||||
item["reviewed_artifact_sha256"] = decision.get("reviewed_artifact_sha256")
|
||||
reviewed_artifact = (
|
||||
Path(item["reviewed_artifact_path"]).expanduser().resolve(strict=False)
|
||||
if isinstance(item.get("reviewed_artifact_path"), str)
|
||||
and item["reviewed_artifact_path"].strip()
|
||||
else None
|
||||
)
|
||||
has_evidence = (
|
||||
item["decision"] == "accepted"
|
||||
and isinstance(item.get("reviewer"), str)
|
||||
and bool(item["reviewer"].strip())
|
||||
and valid_review_timestamp(item.get("reviewed_at"))
|
||||
and isinstance(item.get("reviewed_artifact_path"), str)
|
||||
and bool(item["reviewed_artifact_path"].strip())
|
||||
and valid_sha256(item.get("reviewed_artifact_sha256"))
|
||||
and reviewed_artifact is not None
|
||||
and reviewed_artifact.is_file()
|
||||
and item["reviewed_artifact_sha256"] == sha256(reviewed_artifact)
|
||||
)
|
||||
if has_evidence:
|
||||
reviewed += 1
|
||||
accepted_review_evidence[str(item["sample_slug"])] = {
|
||||
"reviewer": item["reviewer"].strip(),
|
||||
"reviewed_at": item["reviewed_at"],
|
||||
"reviewed_artifact_path": item["reviewed_artifact_path"],
|
||||
"reviewed_artifact_sha256": item["reviewed_artifact_sha256"],
|
||||
}
|
||||
elif item["decision"] in {"accepted", "rejected"}:
|
||||
review_evidence_failures.append(
|
||||
f"{item['sample_slug']}:accepted review lacks reviewer/timestamp/artifact evidence"
|
||||
)
|
||||
review_complete = reviewed == len(review_queue) and all(item["decision"] == "accepted" for item in review_queue)
|
||||
human_review_evidence: dict[str, Any] | None = None
|
||||
if review_decisions_path is not None:
|
||||
human_review_evidence = {
|
||||
"review_decisions_path": str(review_decisions_path),
|
||||
"review_decisions_sha256": sha256(review_decisions_path),
|
||||
"required_sample_count": len(review_queue),
|
||||
"accepted_sample_count": reviewed,
|
||||
"accepted_sample_slugs": sorted(accepted_review_evidence),
|
||||
"reviewer_ids": sorted(
|
||||
{item["reviewer"] for item in accepted_review_evidence.values()}
|
||||
),
|
||||
"reviewed_at_by_sample": {
|
||||
slug: accepted_review_evidence[slug]["reviewed_at"]
|
||||
for slug in sorted(accepted_review_evidence)
|
||||
},
|
||||
"reviewed_artifact_path_by_sample": {
|
||||
slug: accepted_review_evidence[slug]["reviewed_artifact_path"]
|
||||
for slug in sorted(accepted_review_evidence)
|
||||
},
|
||||
"reviewed_artifact_sha256_by_sample": {
|
||||
slug: accepted_review_evidence[slug]["reviewed_artifact_sha256"]
|
||||
for slug in sorted(accepted_review_evidence)
|
||||
},
|
||||
}
|
||||
status = "failed" if failures else ("ok" if review_complete else "needs_human_review")
|
||||
report = {
|
||||
"status": status,
|
||||
"dataset_version": manifest["dataset_version"],
|
||||
"corpus_manifest_path": str(manifest_path.resolve(strict=False)),
|
||||
"corpus_manifest_sha256": sha256(manifest_path),
|
||||
"manifest_immutable": manifest["immutable"],
|
||||
"sample_count": len(samples),
|
||||
"split_counts": {f"{region}/{split}": split_counts[(region, split)] for region in REGIONS for split in REQUIRED_SPLITS},
|
||||
@@ -86,8 +190,13 @@ def main() -> int:
|
||||
"decision_counts": dict(sorted(decision_counts.items())),
|
||||
"temporal_unknown_sample_count": temporal_unknown,
|
||||
"spatial_leakage_status": leakage.get("status"),
|
||||
"training_eligibility_status": manifest_policy.get("status") if isinstance(manifest_policy, dict) else None,
|
||||
"training_eligibility_fixture_mode": fixture_mode,
|
||||
"training_eligibility_failures": eligibility_failures,
|
||||
"reviewed_sample_count": reviewed,
|
||||
"review_complete": review_complete,
|
||||
"human_review_evidence": human_review_evidence,
|
||||
"review_evidence_failures": sorted(set(review_evidence_failures)),
|
||||
"failures": failures,
|
||||
"review_queue": review_queue,
|
||||
}
|
||||
|
||||
@@ -9,7 +9,7 @@ import json
|
||||
import sys
|
||||
from collections import Counter
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from typing import Any, Mapping
|
||||
|
||||
from PIL import Image
|
||||
|
||||
@@ -17,10 +17,21 @@ SCRIPT_DIR = Path(__file__).resolve().parent
|
||||
if str(SCRIPT_DIR) not in sys.path:
|
||||
sys.path.insert(0, str(SCRIPT_DIR))
|
||||
|
||||
from evaluate_belgium_building_candidate import iou, read_references
|
||||
from evaluate_belgium_building_candidate import iou, read_references # noqa: E402
|
||||
from training_dataset_eligibility import ( # noqa: E402
|
||||
TrainingEligibilityError,
|
||||
assert_frozen_manifest_training_eligible,
|
||||
)
|
||||
from training_release_manifest import ( # noqa: E402
|
||||
TrainingReleaseError,
|
||||
assert_yolo_summary_bound_to_embedded_training_release,
|
||||
)
|
||||
|
||||
|
||||
PROTECTED_SPLITS = {"calibration", "test", "background-test"}
|
||||
PROPOSAL_DATASET_PROVENANCE_NAME = "proposal-dataset-provenance.json"
|
||||
PROPOSAL_DATASET_EVIDENCE_NAME = "proposal-dataset-evidence.json"
|
||||
PROPOSAL_DATASET_SCHEMA_VERSION = 1
|
||||
|
||||
|
||||
def sha256(path: Path) -> str:
|
||||
@@ -31,6 +42,125 @@ def sha256(path: Path) -> str:
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def _canonical_json_bytes(payload: Mapping[str, Any]) -> bytes:
|
||||
return json.dumps(payload, ensure_ascii=False, separators=(",", ":"), sort_keys=True).encode("utf-8")
|
||||
|
||||
|
||||
def _immutable_payload_sha256(payload: Mapping[str, Any], *, field: str = "manifest_sha256") -> str:
|
||||
normalized = dict(payload)
|
||||
normalized.pop(field, None)
|
||||
return hashlib.sha256(_canonical_json_bytes(normalized)).hexdigest()
|
||||
|
||||
|
||||
def _write_immutable_json(path: Path, payload: Mapping[str, Any]) -> None:
|
||||
"""Persist one immutable sidecar without silently replacing prior evidence."""
|
||||
|
||||
encoded = json.dumps(dict(payload), ensure_ascii=False, indent=2, sort_keys=True) + "\n"
|
||||
if path.exists():
|
||||
if path.read_text(encoding="utf-8") != encoded:
|
||||
raise RuntimeError(f"immutable provenance artifact already exists with different content: {path}")
|
||||
return
|
||||
path.write_text(encoded, encoding="utf-8")
|
||||
|
||||
|
||||
def assert_summary_source_manifest_binding(summary: Mapping[str, Any], corpus_manifest_path: Path) -> str:
|
||||
"""Require the proposal summary to name the exact frozen corpus bytes."""
|
||||
|
||||
expected = sha256(corpus_manifest_path)
|
||||
observed = summary.get("source_manifest_sha256")
|
||||
if observed != expected:
|
||||
raise ValueError(
|
||||
"proposal source summary is not bound to the supplied governed corpus manifest "
|
||||
f"(expected {expected}, observed {observed!r})"
|
||||
)
|
||||
return expected
|
||||
|
||||
|
||||
def load_governed_corpus_manifest(corpus_manifest_path: Path, *, fixture_mode: bool) -> dict[str, Any]:
|
||||
"""Validate frozen corpus evidence and re-check the live Dataset state.
|
||||
|
||||
This must run before importing Ultralytics/PyTorch so a revocation cannot
|
||||
consume GPU work or create proposal crops.
|
||||
"""
|
||||
|
||||
try:
|
||||
manifest = assert_frozen_manifest_training_eligible(
|
||||
corpus_manifest_path,
|
||||
fixture_mode=fixture_mode,
|
||||
verify_live=True,
|
||||
)
|
||||
except TrainingEligibilityError as exc:
|
||||
raise ValueError(str(exc)) from exc
|
||||
if not isinstance(manifest, dict): # pragma: no cover - defensive contract boundary
|
||||
raise ValueError("governed corpus manifest must be a JSON object")
|
||||
return manifest
|
||||
|
||||
|
||||
def _source_file_evidence(tile: Mapping[str, Any], *, field: str) -> tuple[Path, str]:
|
||||
raw_path = tile.get(field)
|
||||
if not isinstance(raw_path, str) or not raw_path.strip():
|
||||
raise ValueError(f"proposal source tile has no {field}: {tile.get('sample_slug')!r}")
|
||||
path = Path(raw_path).expanduser().resolve(strict=False)
|
||||
if not path.is_file():
|
||||
raise ValueError(f"proposal source tile {field} is unavailable: {path}")
|
||||
return path, sha256(path)
|
||||
|
||||
|
||||
def _source_tile_key(tile: Mapping[str, Any]) -> tuple[str, str, str]:
|
||||
"""Return the canonical source identity used for the pre-inference cache."""
|
||||
|
||||
return (
|
||||
str(tile.get("sample_slug") or ""),
|
||||
str(Path(str(tile.get("image_path") or "")).expanduser().resolve(strict=False)),
|
||||
str(Path(str(tile.get("label_path") or "")).expanduser().resolve(strict=False)),
|
||||
)
|
||||
|
||||
|
||||
def _assert_file_unchanged(path: Path, expected_sha256: str, *, role: str) -> None:
|
||||
"""Reject a mutable source changing between evidence capture and use."""
|
||||
|
||||
if sha256(path) != expected_sha256:
|
||||
raise RuntimeError(f"proposal {role} changed after evidence capture: {path}")
|
||||
|
||||
|
||||
def _crop_entry(
|
||||
*,
|
||||
output_dir: Path,
|
||||
crop_path: Path,
|
||||
tile: Mapping[str, Any],
|
||||
label: str,
|
||||
proposal_index: int,
|
||||
proposal_score: float,
|
||||
source_box_xyxy: tuple[float, float, float, float],
|
||||
source_image_path: Path,
|
||||
source_image_sha256: str,
|
||||
source_label_path: Path,
|
||||
source_label_sha256: str,
|
||||
) -> dict[str, Any]:
|
||||
relative_path = crop_path.resolve(strict=False).relative_to(output_dir.resolve(strict=False)).as_posix()
|
||||
sample_slug = tile.get("sample_slug")
|
||||
split = tile.get("split")
|
||||
if not isinstance(sample_slug, str) or not sample_slug.strip():
|
||||
raise ValueError("proposal source tile has no sample_slug")
|
||||
if split not in {"train", "val"}:
|
||||
raise ValueError(f"proposal crop has unsupported split: {split!r}")
|
||||
return {
|
||||
"relative_path": relative_path,
|
||||
"sha256": sha256(crop_path),
|
||||
"size_bytes": crop_path.stat().st_size,
|
||||
"split": split,
|
||||
"label": label,
|
||||
"sample_slug": sample_slug,
|
||||
"proposal_index": proposal_index,
|
||||
"proposal_score": proposal_score,
|
||||
"source_box_xyxy": [float(value) for value in source_box_xyxy],
|
||||
"source_image_path": str(source_image_path),
|
||||
"source_image_sha256": source_image_sha256,
|
||||
"source_label_path": str(source_label_path),
|
||||
"source_label_sha256": source_label_sha256,
|
||||
}
|
||||
|
||||
|
||||
def classify_proposals(
|
||||
predictions: list[tuple[tuple[float, float, float, float], float]],
|
||||
references: list[tuple[float, float, float, float]],
|
||||
@@ -99,29 +229,74 @@ def main() -> int:
|
||||
parser.add_argument("--max-negative-per-tile", type=int, default=24)
|
||||
parser.add_argument("--device", default="cuda:0")
|
||||
parser.add_argument("--imgsz", type=int, default=640)
|
||||
parser.add_argument(
|
||||
"--fixture-mode",
|
||||
action="store_true",
|
||||
help="Accept only an explicitly fixture-only corpus manifest; never use for operational training data.",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
if args.output_dir.exists():
|
||||
parser.error(f"output already exists: {args.output_dir}")
|
||||
|
||||
from ultralytics import YOLO
|
||||
|
||||
summary = json.loads(args.summary.read_text(encoding="utf-8"))
|
||||
manifest = json.loads(args.corpus_manifest.read_text(encoding="utf-8"))
|
||||
try:
|
||||
summary = json.loads(args.summary.read_text(encoding="utf-8"))
|
||||
except (OSError, json.JSONDecodeError) as exc:
|
||||
raise SystemExit(f"proposal source summary is unreadable: {args.summary}") from exc
|
||||
if not isinstance(summary, dict):
|
||||
raise SystemExit("proposal source summary must be a JSON object")
|
||||
try:
|
||||
source_release = assert_yolo_summary_bound_to_embedded_training_release(
|
||||
summary_path=args.summary,
|
||||
corpus_manifest=args.corpus_manifest,
|
||||
fixture_mode=args.fixture_mode,
|
||||
)
|
||||
manifest = load_governed_corpus_manifest(args.corpus_manifest, fixture_mode=args.fixture_mode)
|
||||
corpus_manifest_sha256 = assert_summary_source_manifest_binding(summary, args.corpus_manifest)
|
||||
except (ValueError, TrainingReleaseError) as exc:
|
||||
raise SystemExit(str(exc)) from exc
|
||||
tiles = eligible_tiles(summary, manifest, args.region)
|
||||
model_path = args.model.expanduser().resolve(strict=False)
|
||||
if not model_path.is_file():
|
||||
raise SystemExit(f"proposal model is unavailable: {model_path}")
|
||||
model_sha256 = sha256(model_path)
|
||||
|
||||
# Hash every input image/label before GPU inference. A crop sidecar then
|
||||
# binds each generated JPEG to the exact source tile rather than its
|
||||
# filename or an unverified aggregate summary.
|
||||
source_evidence: dict[tuple[str, str, str], tuple[Path, str, Path, str]] = {}
|
||||
for tile in tiles:
|
||||
image_path, image_sha256 = _source_file_evidence(tile, field="image_path")
|
||||
label_path, label_sha256 = _source_file_evidence(tile, field="label_path")
|
||||
key = _source_tile_key(tile)
|
||||
source_evidence[key] = (image_path, image_sha256, label_path, label_sha256)
|
||||
|
||||
args.output_dir.mkdir(parents=True)
|
||||
counts: Counter[str] = Counter()
|
||||
sample_counts: Counter[str] = Counter()
|
||||
model = YOLO(str(args.model))
|
||||
crop_entries: list[dict[str, Any]] = []
|
||||
|
||||
# Importing the model is deliberately after the source-manifest and live
|
||||
# Dataset checks above. A revoked corpus must not start GPU work.
|
||||
from ultralytics import YOLO
|
||||
|
||||
model = YOLO(str(model_path))
|
||||
for start in range(0, len(tiles), 16):
|
||||
batch_tiles = tiles[start : start + 16]
|
||||
batch_sources = [source_evidence[_source_tile_key(tile)] for tile in batch_tiles]
|
||||
for image_path, image_sha256, label_path, label_sha256 in batch_sources:
|
||||
_assert_file_unchanged(image_path, image_sha256, role="source image")
|
||||
_assert_file_unchanged(label_path, label_sha256, role="source label")
|
||||
results = model.predict(
|
||||
[tile["image_path"] for tile in batch_tiles], conf=args.confidence,
|
||||
[str(source[0]) for source in batch_sources], conf=args.confidence,
|
||||
device=args.device, imgsz=args.imgsz, max_det=1000, iou=0.7, verbose=False,
|
||||
)
|
||||
for tile, result in zip(batch_tiles, results, strict=True):
|
||||
with Image.open(tile["image_path"]) as opened:
|
||||
image_path, image_sha256, label_path, label_sha256 = source_evidence[_source_tile_key(tile)]
|
||||
_assert_file_unchanged(image_path, image_sha256, role="source image")
|
||||
_assert_file_unchanged(label_path, label_sha256, role="source label")
|
||||
with Image.open(image_path) as opened:
|
||||
source = opened.convert("RGB")
|
||||
references = read_references(Path(tile["label_path"]), source.width, source.height)
|
||||
references = read_references(label_path, source.width, source.height)
|
||||
proposals = [
|
||||
(tuple(map(float, box)), float(score))
|
||||
for box, score in zip(
|
||||
@@ -139,25 +314,113 @@ def main() -> int:
|
||||
target_dir = args.output_dir / split / label
|
||||
target_dir.mkdir(parents=True, exist_ok=True)
|
||||
name = f"{tile['sample_slug']}__{Path(tile['image_path']).stem}__{proposal_index:04d}.jpg"
|
||||
crop_square(source, box, args.crop_scale).save(target_dir / name, quality=92)
|
||||
crop_path = target_dir / name
|
||||
if crop_path.exists():
|
||||
raise RuntimeError(f"proposal crop identity collision: {crop_path}")
|
||||
crop_square(source, box, args.crop_scale).save(crop_path, quality=92)
|
||||
crop_entries.append(
|
||||
_crop_entry(
|
||||
output_dir=args.output_dir,
|
||||
crop_path=crop_path,
|
||||
tile=tile,
|
||||
label=label,
|
||||
proposal_index=proposal_index,
|
||||
proposal_score=score,
|
||||
source_box_xyxy=box,
|
||||
source_image_path=image_path,
|
||||
source_image_sha256=image_sha256,
|
||||
source_label_path=label_path,
|
||||
source_label_sha256=label_sha256,
|
||||
)
|
||||
)
|
||||
counts[f"{split}/{label}"] += 1
|
||||
sample_counts[tile["sample_slug"]] += 1
|
||||
for split in ("train", "val"):
|
||||
for label in ("positive", "negative"):
|
||||
if counts[f"{split}/{label}"] == 0:
|
||||
raise RuntimeError(f"empty proposal class: {split}/{label}")
|
||||
|
||||
crop_entries.sort(key=lambda item: str(item["relative_path"]))
|
||||
if len({str(item["relative_path"]) for item in crop_entries}) != len(crop_entries):
|
||||
raise RuntimeError("proposal crop manifest contains duplicate relative paths")
|
||||
source_summary_sha256 = sha256(args.summary)
|
||||
corpus_freeze_path = args.corpus_manifest.parent / "corpus-freeze.json"
|
||||
if not corpus_freeze_path.is_file(): # guarded above; retain a local invariant for provenance output
|
||||
raise RuntimeError(f"governed corpus freeze is unavailable: {corpus_freeze_path}")
|
||||
provenance: dict[str, Any] = {
|
||||
"schema_version": PROPOSAL_DATASET_SCHEMA_VERSION,
|
||||
"status": "ok",
|
||||
"immutable": True,
|
||||
"dataset_kind": "building_proposal_classifier_crops",
|
||||
"fixture_mode": bool(args.fixture_mode),
|
||||
"governed_corpus_live_recheck": True,
|
||||
"source": {
|
||||
"corpus_manifest": {
|
||||
"path": str(args.corpus_manifest.expanduser().resolve(strict=False)),
|
||||
"sha256": corpus_manifest_sha256,
|
||||
},
|
||||
"corpus_freeze": {
|
||||
"path": str(corpus_freeze_path.resolve(strict=False)),
|
||||
"sha256": sha256(corpus_freeze_path),
|
||||
},
|
||||
"summary": {
|
||||
"path": str(args.summary.expanduser().resolve(strict=False)),
|
||||
"sha256": source_summary_sha256,
|
||||
"source_manifest_sha256": summary["source_manifest_sha256"],
|
||||
"training_release_manifest": summary["training_release_manifest"],
|
||||
"training_release_manifest_sha256": summary["training_release_manifest_sha256"],
|
||||
"training_asset_manifest": summary["training_asset_manifest"],
|
||||
},
|
||||
"training_release": {
|
||||
"dataset_yaml_path": source_release["dataset_yaml"]["path"],
|
||||
"dataset_yaml_sha256": source_release["dataset_yaml"]["sha256"],
|
||||
"corpus_manifest_sha256": source_release["corpus"]["manifest_sha256"],
|
||||
},
|
||||
"proposal_model": {
|
||||
"path": str(model_path),
|
||||
"sha256": model_sha256,
|
||||
},
|
||||
},
|
||||
"parameters": {
|
||||
"region": args.region,
|
||||
"confidence": args.confidence,
|
||||
"match_iou": args.match_iou,
|
||||
"crop_scale": args.crop_scale,
|
||||
"max_positive_per_tile": args.max_positive_per_tile,
|
||||
"max_negative_per_tile": args.max_negative_per_tile,
|
||||
"device": args.device,
|
||||
"imgsz": args.imgsz,
|
||||
},
|
||||
"counts": dict(sorted(counts.items())),
|
||||
"sample_counts": dict(sorted(sample_counts.items())),
|
||||
"tile_count": len(tiles),
|
||||
"crop_count": len(crop_entries),
|
||||
"crops_sha256": hashlib.sha256(_canonical_json_bytes({"crops": crop_entries})).hexdigest(),
|
||||
"crops": crop_entries,
|
||||
}
|
||||
provenance["manifest_sha256"] = _immutable_payload_sha256(provenance)
|
||||
provenance_path = args.output_dir / PROPOSAL_DATASET_PROVENANCE_NAME
|
||||
_write_immutable_json(provenance_path, provenance)
|
||||
|
||||
evidence = {
|
||||
"schema_version": 1, "status": "ok", "model": str(args.model),
|
||||
"model_sha256": sha256(args.model), "summary": str(args.summary),
|
||||
"summary_sha256": sha256(args.summary), "corpus_manifest": str(args.corpus_manifest),
|
||||
"corpus_manifest_sha256": sha256(args.corpus_manifest), "region": args.region,
|
||||
"schema_version": PROPOSAL_DATASET_SCHEMA_VERSION,
|
||||
"status": "ok",
|
||||
"model": str(model_path),
|
||||
"model_sha256": model_sha256,
|
||||
"summary": str(args.summary),
|
||||
"summary_sha256": source_summary_sha256,
|
||||
"corpus_manifest": str(args.corpus_manifest),
|
||||
"corpus_manifest_sha256": corpus_manifest_sha256,
|
||||
"region": args.region,
|
||||
"confidence": args.confidence, "match_iou": args.match_iou, "crop_scale": args.crop_scale,
|
||||
"counts": dict(sorted(counts.items())), "sample_counts": dict(sorted(sample_counts.items())),
|
||||
"protected_samples_in_training": [], "tile_count": len(tiles),
|
||||
"fixture_mode": bool(args.fixture_mode),
|
||||
"proposal_dataset_provenance": str(provenance_path),
|
||||
"proposal_dataset_provenance_sha256": sha256(provenance_path),
|
||||
"proposal_dataset_manifest_sha256": provenance["manifest_sha256"],
|
||||
}
|
||||
(args.output_dir / "proposal-dataset-evidence.json").write_text(
|
||||
json.dumps(evidence, indent=2), encoding="utf-8"
|
||||
)
|
||||
_write_immutable_json(args.output_dir / PROPOSAL_DATASET_EVIDENCE_NAME, evidence)
|
||||
print(json.dumps(evidence, indent=2))
|
||||
return 0
|
||||
|
||||
|
||||
@@ -8,10 +8,25 @@ import hashlib
|
||||
import json
|
||||
import math
|
||||
import re
|
||||
import sys
|
||||
from collections import Counter
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
SCRIPT_DIR = Path(__file__).resolve().parent
|
||||
if str(SCRIPT_DIR) not in sys.path:
|
||||
sys.path.insert(0, str(SCRIPT_DIR))
|
||||
|
||||
from training_dataset_eligibility import ( # noqa: E402
|
||||
TrainingEligibilityError,
|
||||
assert_frozen_manifest_training_eligible,
|
||||
)
|
||||
from training_release_manifest import ( # noqa: E402
|
||||
TrainingReleaseError,
|
||||
assert_yolo_summary_bound_to_embedded_training_release,
|
||||
create_training_release_manifest,
|
||||
)
|
||||
|
||||
|
||||
PRECISION_NEGATIVE_CONTEXTS = {
|
||||
"coastal-urban": {"port-hard-negative", "dunes-negative"},
|
||||
@@ -80,6 +95,17 @@ def build_sampling(
|
||||
) -> tuple[list[str], dict[str, Any]]:
|
||||
if assessment.get("status") != "continue_training_loop":
|
||||
raise ValueError("Failure-driven sampling requires a failed assessment")
|
||||
protected_feedback = [
|
||||
role
|
||||
for role in ("test", "background")
|
||||
if assessment.get(role) is not None
|
||||
]
|
||||
if protected_feedback:
|
||||
raise ValueError(
|
||||
"Failure-driven sampling is prohibited after protected "
|
||||
+ "/".join(protected_feedback)
|
||||
+ " evidence was opened"
|
||||
)
|
||||
if min(
|
||||
positive_repeat,
|
||||
negative_repeat,
|
||||
@@ -97,9 +123,9 @@ def build_sampling(
|
||||
|
||||
samples = {item["sample_slug"]: item for item in manifest["samples"]}
|
||||
gates = assessment["gates"]
|
||||
evaluation = assessment.get("test") or assessment.get("calibration")
|
||||
evaluation = assessment.get("calibration")
|
||||
if not evaluation or "regions" not in evaluation:
|
||||
raise ValueError("Assessment has no regional calibration or test evidence")
|
||||
raise ValueError("Assessment has no regional calibration evidence")
|
||||
regions = evaluation["regions"]
|
||||
weak_recall_regions = {
|
||||
region
|
||||
@@ -246,7 +272,7 @@ def build_sampling(
|
||||
"schema_version": 1,
|
||||
"status": "ok",
|
||||
"strategy": "failed-region-positive-and-hard-negative-repeat",
|
||||
"failure_evidence_source": "test" if assessment.get("test") else "calibration",
|
||||
"failure_evidence_source": "calibration",
|
||||
"weak_recall_regions": sorted(weak_recall_regions),
|
||||
"weak_precision_regions": sorted(weak_precision_regions),
|
||||
"recall_dominant_regions": sorted(recall_dominant_regions),
|
||||
@@ -290,6 +316,11 @@ def main() -> int:
|
||||
parser.add_argument("--corpus-manifest", type=Path, required=True)
|
||||
parser.add_argument("--assessment", type=Path, required=True)
|
||||
parser.add_argument("--output-dir", type=Path, required=True)
|
||||
parser.add_argument(
|
||||
"--review-audit",
|
||||
type=Path,
|
||||
help="Passed corpus audit containing accepted human-review evidence for the frozen corpus.",
|
||||
)
|
||||
parser.add_argument("--positive-repeat", type=int, default=3)
|
||||
parser.add_argument("--negative-repeat", type=int, default=4)
|
||||
parser.add_argument("--precision-positive-repeat", type=int, default=1)
|
||||
@@ -299,8 +330,26 @@ def main() -> int:
|
||||
parser.add_argument("--sampling-round", type=int)
|
||||
parser.add_argument("--precision-guard-band", type=float, default=0.03)
|
||||
parser.add_argument("--recall-guard-band", type=float, default=0.03)
|
||||
parser.add_argument(
|
||||
"--fixture-mode",
|
||||
action="store_true",
|
||||
help="Accept only an explicitly fixture-only corpus manifest; never use for operational sampling.",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
try:
|
||||
assert_frozen_manifest_training_eligible(
|
||||
args.corpus_manifest,
|
||||
fixture_mode=args.fixture_mode,
|
||||
verify_live=True,
|
||||
)
|
||||
source_release = assert_yolo_summary_bound_to_embedded_training_release(
|
||||
summary_path=args.summary,
|
||||
corpus_manifest=args.corpus_manifest,
|
||||
fixture_mode=args.fixture_mode,
|
||||
)
|
||||
except (TrainingEligibilityError, TrainingReleaseError) as exc:
|
||||
raise SystemExit(str(exc)) from exc
|
||||
summary = json.loads(args.summary.read_text(encoding="utf-8"))
|
||||
manifest = json.loads(args.corpus_manifest.read_text(encoding="utf-8"))
|
||||
assessment = json.loads(args.assessment.read_text(encoding="utf-8"))
|
||||
@@ -325,7 +374,7 @@ def main() -> int:
|
||||
args.output_dir.mkdir(parents=True, exist_ok=True)
|
||||
train_list = args.output_dir / "train-failure-driven.txt"
|
||||
train_list.write_text("\n".join(paths) + "\n", encoding="utf-8")
|
||||
source_yaml = args.summary.parent / "dataset.yaml"
|
||||
source_yaml = Path(str(source_release["dataset_yaml"]["path"]))
|
||||
val_source = dataset_validation_source(source_yaml)
|
||||
dataset_yaml = args.output_dir / "dataset.yaml"
|
||||
dataset_yaml.write_text(
|
||||
@@ -335,6 +384,15 @@ def main() -> int:
|
||||
"names:\n 0: building\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
try:
|
||||
release_paths = create_training_release_manifest(
|
||||
train_yaml=dataset_yaml,
|
||||
corpus_manifest=args.corpus_manifest,
|
||||
review_audit_path=args.review_audit,
|
||||
fixture_mode=args.fixture_mode,
|
||||
)
|
||||
except TrainingReleaseError as exc:
|
||||
raise SystemExit(str(exc)) from exc
|
||||
metadata.update(
|
||||
{
|
||||
"summary": str(args.summary),
|
||||
@@ -346,6 +404,11 @@ def main() -> int:
|
||||
"source_dataset_yaml": str(source_yaml),
|
||||
"train_list": str(train_list),
|
||||
"dataset_yaml": str(dataset_yaml),
|
||||
"training_release_manifest": str(release_paths["release_manifest"]),
|
||||
"training_release_manifest_sha256": file_sha256(release_paths["release_manifest"]),
|
||||
"training_release_freeze": str(release_paths["release_freeze"]),
|
||||
"training_asset_manifest": str(release_paths["asset_manifest"]),
|
||||
"fixture_mode": bool(args.fixture_mode),
|
||||
}
|
||||
)
|
||||
output = args.output_dir / "failure-driven-sampling.json"
|
||||
|
||||
@@ -7,11 +7,24 @@ import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import shutil
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
from PIL import Image
|
||||
|
||||
|
||||
SCRIPT_DIR = Path(__file__).resolve().parent
|
||||
if str(SCRIPT_DIR) not in sys.path:
|
||||
sys.path.insert(0, str(SCRIPT_DIR))
|
||||
|
||||
from training_release_manifest import ( # noqa: E402
|
||||
TrainingReleaseError,
|
||||
assert_yolo_summary_bound_to_training_release,
|
||||
file_sha256,
|
||||
training_release_paths,
|
||||
)
|
||||
|
||||
|
||||
def sha256(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as stream:
|
||||
@@ -23,9 +36,25 @@ def sha256(path: Path) -> str:
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--summary", type=Path, required=True)
|
||||
parser.add_argument("--train-yaml", type=Path, required=True)
|
||||
parser.add_argument("--corpus-manifest", type=Path, required=True)
|
||||
parser.add_argument("--output-dir", type=Path, required=True)
|
||||
parser.add_argument(
|
||||
"--fixture-mode",
|
||||
action="store_true",
|
||||
help="Accept only an explicitly fixture-only release; never creates a production-ready derived dataset.",
|
||||
)
|
||||
parser.add_argument("--force", action="store_true")
|
||||
args = parser.parse_args()
|
||||
try:
|
||||
release = assert_yolo_summary_bound_to_training_release(
|
||||
summary_path=args.summary,
|
||||
train_yaml=args.train_yaml,
|
||||
corpus_manifest=args.corpus_manifest,
|
||||
fixture_mode=args.fixture_mode,
|
||||
)
|
||||
except TrainingReleaseError as exc:
|
||||
raise SystemExit(str(exc)) from exc
|
||||
if args.output_dir.exists():
|
||||
if not args.force:
|
||||
raise SystemExit(f"Output exists: {args.output_dir}")
|
||||
@@ -54,6 +83,16 @@ def main() -> int:
|
||||
f"path: {args.output_dir}\ntrain: images/train\nval: images/val\nnames:\n 0: building\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
# The source release binds the original image bytes. These transformed
|
||||
# bytes cannot inherit it, so remove its operational release pointers and
|
||||
# retain them only as explicitly non-trainable parent evidence below.
|
||||
for field_name in (
|
||||
"training_release_manifest",
|
||||
"training_release_manifest_sha256",
|
||||
"training_release_freeze",
|
||||
"training_asset_manifest",
|
||||
):
|
||||
summary.pop(field_name, None)
|
||||
summary.update(
|
||||
{
|
||||
"output_dir": str(args.output_dir),
|
||||
@@ -71,10 +110,19 @@ def main() -> int:
|
||||
"preprocessing": "luminance_rgb_replicated",
|
||||
"source_summary": str(args.summary),
|
||||
"source_summary_sha256": sha256(args.summary),
|
||||
"source_training_release": str(training_release_paths(args.train_yaml)["release_manifest"]),
|
||||
"source_training_release_sha256": file_sha256(
|
||||
training_release_paths(args.train_yaml)["release_manifest"]
|
||||
),
|
||||
"source_corpus_manifest_sha256": release["corpus"]["manifest_sha256"],
|
||||
"converted_tile_count": converted,
|
||||
"output_summary": str(output_summary),
|
||||
"output_summary_sha256": sha256(output_summary),
|
||||
"dataset_yaml": str(dataset_yaml),
|
||||
"training_eligible": False,
|
||||
"training_eligibility_reason": (
|
||||
"Derived image bytes require a new governed corpus, validation report and immutable training release."
|
||||
),
|
||||
}
|
||||
(args.output_dir / "grayscale-dataset-evidence.json").write_text(
|
||||
json.dumps(evidence, indent=2), encoding="utf-8"
|
||||
|
||||
@@ -6,10 +6,25 @@ from __future__ import annotations
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import sys
|
||||
from collections import Counter
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
SCRIPT_DIR = Path(__file__).resolve().parent
|
||||
if str(SCRIPT_DIR) not in sys.path:
|
||||
sys.path.insert(0, str(SCRIPT_DIR))
|
||||
|
||||
from training_dataset_eligibility import ( # noqa: E402
|
||||
TrainingEligibilityError,
|
||||
assert_frozen_manifest_training_eligible,
|
||||
)
|
||||
from training_release_manifest import ( # noqa: E402
|
||||
TrainingReleaseError,
|
||||
assert_yolo_summary_bound_to_embedded_training_release,
|
||||
create_training_release_manifest,
|
||||
)
|
||||
|
||||
|
||||
PROTECTED_SPLITS = {"calibration", "test", "background-test"}
|
||||
|
||||
@@ -138,11 +153,34 @@ def main() -> int:
|
||||
parser.add_argument("--priority-context", action="append", default=[])
|
||||
parser.add_argument("--priority-repeat", type=int, default=2)
|
||||
parser.add_argument("--negative-repeat", type=int, default=2)
|
||||
parser.add_argument(
|
||||
"--review-audit",
|
||||
type=Path,
|
||||
help="Accepted corpus-audit evidence; mandatory for an operational training release.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--fixture-mode",
|
||||
action="store_true",
|
||||
help="Accept only an explicitly fixture-only corpus manifest; never use for operational training data.",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
if args.priority_repeat < 1 or args.negative_repeat < 1:
|
||||
parser.error("repeat counts must be positive")
|
||||
summary = json.loads(args.summary.read_text(encoding="utf-8"))
|
||||
manifest = json.loads(args.corpus_manifest.read_text(encoding="utf-8"))
|
||||
try:
|
||||
assert_frozen_manifest_training_eligible(
|
||||
args.corpus_manifest,
|
||||
fixture_mode=args.fixture_mode,
|
||||
verify_live=True,
|
||||
)
|
||||
assert_yolo_summary_bound_to_embedded_training_release(
|
||||
summary_path=args.summary,
|
||||
corpus_manifest=args.corpus_manifest,
|
||||
fixture_mode=args.fixture_mode,
|
||||
)
|
||||
except (TrainingEligibilityError, TrainingReleaseError) as exc:
|
||||
raise SystemExit(str(exc)) from exc
|
||||
train, val, evidence = build(
|
||||
summary=summary,
|
||||
manifest=manifest,
|
||||
@@ -161,6 +199,15 @@ def main() -> int:
|
||||
f"path: /\ntrain: {train_path}\nval: {val_path}\nnames:\n 0: building\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
try:
|
||||
release_paths = create_training_release_manifest(
|
||||
train_yaml=yaml_path,
|
||||
corpus_manifest=args.corpus_manifest,
|
||||
review_audit_path=args.review_audit,
|
||||
fixture_mode=args.fixture_mode,
|
||||
)
|
||||
except TrainingReleaseError as exc:
|
||||
raise SystemExit(str(exc)) from exc
|
||||
evidence.update({
|
||||
"source_summary": str(args.summary),
|
||||
"source_summary_sha256": sha256(args.summary),
|
||||
@@ -169,6 +216,11 @@ def main() -> int:
|
||||
"train_sha256": sha256(train_path),
|
||||
"validation_sha256": sha256(val_path),
|
||||
"dataset_yaml": str(yaml_path),
|
||||
"training_release_manifest": str(release_paths["release_manifest"]),
|
||||
"training_release_manifest_sha256": sha256(release_paths["release_manifest"]),
|
||||
"training_release_freeze": str(release_paths["release_freeze"]),
|
||||
"training_asset_manifest": str(release_paths["asset_manifest"]),
|
||||
"fixture_mode": bool(args.fixture_mode),
|
||||
})
|
||||
encoded_evidence = json.dumps(evidence, indent=2)
|
||||
(args.output_dir / "regional-dataset-evidence.json").write_text(encoded_evidence, encoding="utf-8")
|
||||
|
||||
@@ -6,8 +6,23 @@ from __future__ import annotations
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
SCRIPT_DIR = Path(__file__).resolve().parent
|
||||
if str(SCRIPT_DIR) not in sys.path:
|
||||
sys.path.insert(0, str(SCRIPT_DIR))
|
||||
|
||||
from training_dataset_eligibility import ( # noqa: E402
|
||||
TrainingEligibilityError,
|
||||
assert_frozen_manifest_training_eligible,
|
||||
)
|
||||
from training_release_manifest import ( # noqa: E402
|
||||
TrainingReleaseError,
|
||||
assert_yolo_summary_bound_to_embedded_training_release,
|
||||
create_training_release_manifest,
|
||||
)
|
||||
|
||||
|
||||
def sha256(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
@@ -31,12 +46,35 @@ def main() -> int:
|
||||
default=0,
|
||||
help="Include each train tile outside the expert region this many times.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--fixture-mode",
|
||||
action="store_true",
|
||||
help="Accept only an explicitly fixture-only corpus manifest; never use for operational training data.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--review-audit",
|
||||
type=Path,
|
||||
help="Accepted corpus-audit evidence; mandatory for an operational training release.",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
if args.positive_repeat < 1 or args.negative_repeat < 1 or args.other_region_repeat < 0:
|
||||
raise SystemExit("regional repeats must be positive and other-region repeat non-negative")
|
||||
|
||||
summary = json.loads(args.summary.read_text(encoding="utf-8"))
|
||||
manifest = json.loads(args.corpus_manifest.read_text(encoding="utf-8"))
|
||||
try:
|
||||
assert_frozen_manifest_training_eligible(
|
||||
args.corpus_manifest,
|
||||
fixture_mode=args.fixture_mode,
|
||||
verify_live=True,
|
||||
)
|
||||
assert_yolo_summary_bound_to_embedded_training_release(
|
||||
summary_path=args.summary,
|
||||
corpus_manifest=args.corpus_manifest,
|
||||
fixture_mode=args.fixture_mode,
|
||||
)
|
||||
except (TrainingEligibilityError, TrainingReleaseError) as exc:
|
||||
raise SystemExit(str(exc)) from exc
|
||||
samples = {item["sample_slug"]: item for item in manifest["samples"]}
|
||||
paths: list[str] = []
|
||||
selected_samples: set[str] = set()
|
||||
@@ -69,6 +107,15 @@ def main() -> int:
|
||||
f"val: {args.summary.parent / 'images' / 'val'}\nnames:\n 0: building\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
try:
|
||||
release_paths = create_training_release_manifest(
|
||||
train_yaml=dataset_yaml,
|
||||
corpus_manifest=args.corpus_manifest,
|
||||
review_audit_path=args.review_audit,
|
||||
fixture_mode=args.fixture_mode,
|
||||
)
|
||||
except TrainingReleaseError as exc:
|
||||
raise SystemExit(str(exc)) from exc
|
||||
evidence = {
|
||||
"schema_version": 1,
|
||||
"status": "ok",
|
||||
@@ -88,6 +135,11 @@ def main() -> int:
|
||||
"protected_samples_in_training": [],
|
||||
"train_list": str(train_list),
|
||||
"dataset_yaml": str(dataset_yaml),
|
||||
"training_release_manifest": str(release_paths["release_manifest"]),
|
||||
"training_release_manifest_sha256": sha256(release_paths["release_manifest"]),
|
||||
"training_release_freeze": str(release_paths["release_freeze"]),
|
||||
"training_asset_manifest": str(release_paths["asset_manifest"]),
|
||||
"fixture_mode": bool(args.fixture_mode),
|
||||
}
|
||||
evidence_path = args.output_dir / "regional-expert-dataset.json"
|
||||
evidence_path.write_text(json.dumps(evidence, indent=2), encoding="utf-8")
|
||||
|
||||
@@ -9,6 +9,7 @@ new provider data.
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
@@ -16,6 +17,19 @@ import sys
|
||||
from pathlib import Path
|
||||
from typing import Any, Iterable
|
||||
|
||||
SCRIPT_DIR = Path(__file__).resolve().parent
|
||||
if str(SCRIPT_DIR) not in sys.path:
|
||||
sys.path.insert(0, str(SCRIPT_DIR))
|
||||
|
||||
from training_dataset_eligibility import ( # noqa: E402
|
||||
TrainingEligibilityError,
|
||||
assert_frozen_manifest_training_eligible,
|
||||
)
|
||||
from training_release_manifest import ( # noqa: E402
|
||||
TrainingReleaseError,
|
||||
create_training_release_manifest,
|
||||
)
|
||||
|
||||
|
||||
DEFAULT_MANIFEST_PATH = Path("/app/storage/operator-data/operator_samples_manifest.json")
|
||||
DEFAULT_OUTPUT_DIR = Path("/app/storage/operator-data/yolo-building-dataset")
|
||||
@@ -24,6 +38,14 @@ Transformer: Any = None
|
||||
Image: Any = None
|
||||
|
||||
|
||||
def file_sha256(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as stream:
|
||||
for chunk in iter(lambda: stream.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Export operator real-data samples to a YOLO detection dataset.",
|
||||
@@ -45,6 +67,12 @@ def parse_args() -> argparse.Namespace:
|
||||
default=os.environ.get("OPERATOR_YOLO_VAL_SAMPLES", "turnhout"),
|
||||
help="Comma/space separated sample slugs assigned to validation. Defaults to turnhout.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--review-audit",
|
||||
type=Path,
|
||||
required=True,
|
||||
help="Passed corpus audit with accepted human-review evidence for this frozen source manifest.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--force",
|
||||
action="store_true",
|
||||
@@ -218,12 +246,16 @@ def ensure_yolo_directories(output_dir: Path) -> None:
|
||||
|
||||
def main() -> int:
|
||||
args = parse_args()
|
||||
manifest = json.loads(args.manifest_path.read_text(encoding="utf-8-sig"))
|
||||
try:
|
||||
assert_frozen_manifest_training_eligible(args.manifest_path, verify_live=True)
|
||||
except TrainingEligibilityError as exc:
|
||||
raise SystemExit(str(exc)) from exc
|
||||
ensure_dependencies()
|
||||
if args.force and args.output_dir.exists():
|
||||
shutil.rmtree(args.output_dir)
|
||||
args.output_dir.mkdir(parents=True, exist_ok=True)
|
||||
ensure_yolo_directories(args.output_dir)
|
||||
manifest = json.loads(args.manifest_path.read_text(encoding="utf-8-sig"))
|
||||
samples = manifest.get("samples") or []
|
||||
if not samples:
|
||||
raise SystemExit("Operator sample manifest contains no samples")
|
||||
@@ -234,12 +266,26 @@ def main() -> int:
|
||||
if not any(item["split"] == "val" for item in exported):
|
||||
raise SystemExit("YOLO dataset export produced no validation samples")
|
||||
dataset_yaml = write_dataset_yaml(args.output_dir)
|
||||
try:
|
||||
release_paths = create_training_release_manifest(
|
||||
train_yaml=dataset_yaml,
|
||||
corpus_manifest=args.manifest_path,
|
||||
review_audit_path=args.review_audit,
|
||||
)
|
||||
except TrainingReleaseError as exc:
|
||||
raise SystemExit(str(exc)) from exc
|
||||
summary = {
|
||||
"status": "ok",
|
||||
"dataset_yaml": str(dataset_yaml),
|
||||
"training_release_manifest": str(release_paths["release_manifest"]),
|
||||
"training_release_manifest_sha256": file_sha256(release_paths["release_manifest"]),
|
||||
"training_release_freeze": str(release_paths["release_freeze"]),
|
||||
"training_asset_manifest": str(release_paths["asset_manifest"]),
|
||||
"output_dir": str(args.output_dir),
|
||||
"class_names": ["building"],
|
||||
"sample_count": len(exported),
|
||||
"source_manifest": str(args.manifest_path),
|
||||
"source_manifest_sha256": file_sha256(args.manifest_path),
|
||||
"train_sample_count": sum(1 for item in exported if item["split"] == "train"),
|
||||
"val_sample_count": sum(1 for item in exported if item["split"] == "val"),
|
||||
"label_count": sum(item["label_count"] for item in exported),
|
||||
|
||||
@@ -18,6 +18,19 @@ import sys
|
||||
from pathlib import Path
|
||||
from typing import Any, Iterable
|
||||
|
||||
SCRIPT_DIR = Path(__file__).resolve().parent
|
||||
if str(SCRIPT_DIR) not in sys.path:
|
||||
sys.path.insert(0, str(SCRIPT_DIR))
|
||||
|
||||
from training_dataset_eligibility import ( # noqa: E402
|
||||
TrainingEligibilityError,
|
||||
assert_frozen_manifest_training_eligible,
|
||||
)
|
||||
from training_release_manifest import ( # noqa: E402
|
||||
TrainingReleaseError,
|
||||
create_training_release_manifest,
|
||||
)
|
||||
|
||||
|
||||
DEFAULT_MANIFEST_PATH = Path("/app/storage/operator-data/operator_samples_manifest.json")
|
||||
DEFAULT_OUTPUT_DIR = Path("/app/storage/operator-data/yolo-building-tile-dataset")
|
||||
@@ -45,6 +58,14 @@ Transformer: Any = None
|
||||
Image: Any = None
|
||||
|
||||
|
||||
def file_sha256(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as stream:
|
||||
for chunk in iter(lambda: stream.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class TileWindow:
|
||||
row_off: int
|
||||
@@ -87,6 +108,12 @@ def parse_args() -> argparse.Namespace:
|
||||
default=os.environ.get("OPERATOR_YOLO_REFERENCE_SOURCE", DEFAULT_REFERENCE_SOURCE),
|
||||
help="Required source_name in reference GeoJSON features.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--review-audit",
|
||||
type=Path,
|
||||
required=True,
|
||||
help="Passed corpus audit with accepted human-review evidence for this frozen source manifest.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--reference-layer",
|
||||
default=os.environ.get("OPERATOR_YOLO_REFERENCE_LAYER", DEFAULT_REFERENCE_LAYER),
|
||||
@@ -626,13 +653,17 @@ def main() -> int:
|
||||
):
|
||||
if not value or any(character not in "abcdefghijklmnopqrstuvwxyz0123456789_-" for character in value):
|
||||
raise SystemExit(f"YOLO {label} must be a non-empty canonical slug")
|
||||
manifest = json.loads(args.manifest_path.read_text(encoding="utf-8-sig"))
|
||||
try:
|
||||
assert_frozen_manifest_training_eligible(args.manifest_path, verify_live=True)
|
||||
except TrainingEligibilityError as exc:
|
||||
raise SystemExit(str(exc)) from exc
|
||||
ensure_dependencies()
|
||||
if args.force and args.output_dir.exists():
|
||||
shutil.rmtree(args.output_dir)
|
||||
args.output_dir.mkdir(parents=True, exist_ok=True)
|
||||
ensure_yolo_directories(args.output_dir)
|
||||
|
||||
manifest = json.loads(args.manifest_path.read_text(encoding="utf-8-sig"))
|
||||
manifest_samples = manifest.get("samples") or []
|
||||
if not manifest_samples:
|
||||
raise SystemExit("Operator sample manifest contains no samples")
|
||||
@@ -672,6 +703,14 @@ def main() -> int:
|
||||
if not args.allow_empty_validation and not any(tile["split"] == "val" for tile in kept_tiles):
|
||||
raise SystemExit("YOLO tile dataset export produced no validation tiles")
|
||||
dataset_yaml = write_dataset_yaml(args.output_dir, class_name)
|
||||
try:
|
||||
release_paths = create_training_release_manifest(
|
||||
train_yaml=dataset_yaml,
|
||||
corpus_manifest=args.manifest_path,
|
||||
review_audit_path=args.review_audit,
|
||||
)
|
||||
except TrainingReleaseError as exc:
|
||||
raise SystemExit(str(exc)) from exc
|
||||
positive_tiles = [tile for tile in kept_tiles if not tile["is_negative"]]
|
||||
negative_tiles = [tile for tile in kept_tiles if tile["is_negative"]]
|
||||
skipped_negative_tiles = [tile for tile in exported_tiles if not tile["kept"] and tile["is_negative"]]
|
||||
@@ -684,6 +723,10 @@ def main() -> int:
|
||||
summary = {
|
||||
"status": "ok",
|
||||
"dataset_yaml": str(dataset_yaml),
|
||||
"training_release_manifest": str(release_paths["release_manifest"]),
|
||||
"training_release_manifest_sha256": file_sha256(release_paths["release_manifest"]),
|
||||
"training_release_freeze": str(release_paths["release_freeze"]),
|
||||
"training_asset_manifest": str(release_paths["asset_manifest"]),
|
||||
"output_dir": str(args.output_dir),
|
||||
"class_names": [class_name],
|
||||
"reference_source": reference_source,
|
||||
@@ -697,6 +740,8 @@ def main() -> int:
|
||||
"drop_low_variance_negatives": args.drop_low_variance_negatives,
|
||||
"blank_range_threshold": args.blank_range_threshold,
|
||||
"source_manifest_sample_count": len(manifest_samples),
|
||||
"source_manifest": str(args.manifest_path),
|
||||
"source_manifest_sha256": file_sha256(args.manifest_path),
|
||||
"source_sample_count": len(samples),
|
||||
"selected_sample_slugs": sorted(
|
||||
str(sample.get("sample_slug") or "").strip().lower() for sample in samples
|
||||
|
||||
@@ -16,6 +16,12 @@ from dataclasses import dataclass, replace
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
SCRIPT_DIR = Path(__file__).resolve().parent
|
||||
if str(SCRIPT_DIR) not in sys.path:
|
||||
sys.path.insert(0, str(SCRIPT_DIR))
|
||||
|
||||
from training_dataset_eligibility import TRAINING_ELIGIBILITY_POLICY_VERSION # noqa: E402
|
||||
|
||||
|
||||
WMS_URL = "https://geo.api.vlaanderen.be/omwrgbmrvl/wms"
|
||||
GRB_GBG_URL = "https://geo.api.vlaanderen.be/GRB/ogc/features/v1/collections/GBG/items"
|
||||
@@ -803,6 +809,14 @@ def main() -> int:
|
||||
manifest = {
|
||||
"schema_version": 2,
|
||||
"description": "GeoIntel operator real-data samples for configured-YOLO QA validation.",
|
||||
"training_eligibility": {
|
||||
"policy_version": TRAINING_ELIGIBILITY_POLICY_VERSION,
|
||||
"status": "not_eligible",
|
||||
"reason": (
|
||||
"Direct provider downloads are QA-only until re-ingested through the governed "
|
||||
"dataset source registry, contract validator and provenance snapshot flow."
|
||||
),
|
||||
},
|
||||
"output_dir": str(output_dir),
|
||||
"sample_width": args.width,
|
||||
"sample_height": args.height,
|
||||
|
||||
@@ -626,7 +626,7 @@ def provision_dataset(
|
||||
from app.services.dataset_service import DatasetService
|
||||
|
||||
partition_checksums = {
|
||||
summary["nis_code"]: summary["sha256"]
|
||||
summary["filename"]: summary["sha256"]
|
||||
for summary in manifest["partitions"]
|
||||
}
|
||||
metadata_json = {
|
||||
|
||||
@@ -43,7 +43,6 @@ from provision_regional_grb_buildings import (
|
||||
list_paginated_items,
|
||||
next_page_url,
|
||||
observed_at,
|
||||
response_data,
|
||||
reusable_manifest,
|
||||
safe_slug,
|
||||
sha256_file,
|
||||
@@ -758,7 +757,7 @@ def provision_dataset(
|
||||
"source_urls": manifest["grb_source_urls"],
|
||||
"artifact_sha256": manifest["artifact_sha256"],
|
||||
"artifact_size_bytes": manifest["artifact_size_bytes"],
|
||||
"partition_checksums": {summary["nis_code"]: summary["sha256"] for summary in manifest["partitions"]},
|
||||
"partition_checksums": {summary["filename"]: summary["sha256"] for summary in manifest["partitions"]},
|
||||
"partition_assignment_rule": manifest["partition_assignment_rule"],
|
||||
"reference_truncated": False,
|
||||
}
|
||||
|
||||
@@ -10,8 +10,20 @@ import json
|
||||
import math
|
||||
import os
|
||||
import shutil
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
SCRIPT_DIR = Path(__file__).resolve().parent
|
||||
if str(SCRIPT_DIR) not in sys.path:
|
||||
sys.path.insert(0, str(SCRIPT_DIR))
|
||||
|
||||
from training_release_manifest import ( # noqa: E402
|
||||
TrainingReleaseError,
|
||||
assert_yolo_summary_bound_to_training_release,
|
||||
file_sha256,
|
||||
training_release_paths,
|
||||
)
|
||||
|
||||
|
||||
def sha256(path: Path) -> str:
|
||||
@@ -97,6 +109,8 @@ def link_or_copy(source: Path, target: Path) -> None:
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--summary", type=Path, required=True)
|
||||
parser.add_argument("--train-yaml", type=Path, required=True)
|
||||
parser.add_argument("--corpus-manifest", type=Path, required=True)
|
||||
parser.add_argument("--model", type=Path, required=True)
|
||||
parser.add_argument("--output-dir", type=Path, required=True)
|
||||
parser.add_argument("--device", default="cuda:0")
|
||||
@@ -110,7 +124,21 @@ def main() -> int:
|
||||
parser.add_argument("--max-prompts-per-pass", type=int, default=96)
|
||||
parser.add_argument("--fallback-policy", choices=("retain", "drop"), default="retain")
|
||||
parser.add_argument("--force", action="store_true")
|
||||
parser.add_argument(
|
||||
"--fixture-mode",
|
||||
action="store_true",
|
||||
help="Accept only an explicitly fixture-only source release; refined labels remain non-trainable.",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
try:
|
||||
source_release = assert_yolo_summary_bound_to_training_release(
|
||||
summary_path=args.summary,
|
||||
train_yaml=args.train_yaml,
|
||||
corpus_manifest=args.corpus_manifest,
|
||||
fixture_mode=args.fixture_mode,
|
||||
)
|
||||
except TrainingReleaseError as exc:
|
||||
raise SystemExit(str(exc)) from exc
|
||||
if args.output_dir.exists():
|
||||
if not args.force:
|
||||
raise SystemExit(f"Output exists: {args.output_dir}")
|
||||
@@ -194,6 +222,16 @@ def main() -> int:
|
||||
print(f"{index}/{len(summary['tiles'])} {tile['sample_slug']}: {len(source_boxes)}", flush=True)
|
||||
|
||||
output_summary = dict(summary)
|
||||
# SAM changes label bytes and semantics. It therefore cannot inherit the
|
||||
# source release; keep the parent evidence separately and require a new
|
||||
# governed corpus/review/release before any training consumer can use it.
|
||||
for field_name in (
|
||||
"training_release_manifest",
|
||||
"training_release_manifest_sha256",
|
||||
"training_release_freeze",
|
||||
"training_asset_manifest",
|
||||
):
|
||||
output_summary.pop(field_name, None)
|
||||
output_summary.update(
|
||||
{
|
||||
"output_dir": str(args.output_dir),
|
||||
@@ -207,6 +245,10 @@ def main() -> int:
|
||||
"label_count": refined_count if args.fallback_policy == "drop" else summary["label_count"],
|
||||
"positive_tile_count": sum(not tile["is_negative"] for tile in output_tiles),
|
||||
"negative_tile_count": sum(tile["is_negative"] for tile in output_tiles),
|
||||
"training_eligible": False,
|
||||
"training_eligibility_reason": (
|
||||
"SAM-refined labels require a new governed corpus, validation report, human review and immutable training release."
|
||||
),
|
||||
}
|
||||
)
|
||||
summary_path = args.output_dir / "yolo_tile_dataset_summary.json"
|
||||
@@ -220,6 +262,11 @@ def main() -> int:
|
||||
"status": "ok",
|
||||
"source_summary": str(args.summary),
|
||||
"source_summary_sha256": sha256(args.summary),
|
||||
"source_training_release": str(training_release_paths(args.train_yaml)["release_manifest"]),
|
||||
"source_training_release_sha256": file_sha256(
|
||||
training_release_paths(args.train_yaml)["release_manifest"]
|
||||
),
|
||||
"source_corpus_manifest_sha256": source_release["corpus"]["manifest_sha256"],
|
||||
"sam_model": str(args.model),
|
||||
"sam_model_sha256": sha256(args.model),
|
||||
"device": args.device,
|
||||
@@ -236,6 +283,8 @@ def main() -> int:
|
||||
"fallback_label_count": fallback_count,
|
||||
"dropped_fallback_label_count": fallback_count if args.fallback_policy == "drop" else 0,
|
||||
"fallback_reason_counts": reason_counts,
|
||||
"training_eligible": False,
|
||||
"fixture_mode": bool(args.fixture_mode),
|
||||
}
|
||||
(args.output_dir / "sam-refinement.json").write_text(json.dumps(evidence, indent=2), encoding="utf-8")
|
||||
print(json.dumps(evidence, indent=2))
|
||||
|
||||
@@ -7,6 +7,7 @@ import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import shutil
|
||||
import sys
|
||||
from collections import Counter
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
@@ -15,6 +16,19 @@ from pyproj import Transformer
|
||||
from shapely.geometry import box
|
||||
from shapely.ops import transform as shapely_transform
|
||||
|
||||
SCRIPT_DIR = Path(__file__).resolve().parent
|
||||
if str(SCRIPT_DIR) not in sys.path:
|
||||
sys.path.insert(0, str(SCRIPT_DIR))
|
||||
|
||||
from training_dataset_eligibility import ( # noqa: E402
|
||||
TrainingEligibilityError,
|
||||
assert_frozen_manifest_training_eligible,
|
||||
)
|
||||
from training_release_manifest import ( # noqa: E402
|
||||
TrainingReleaseError,
|
||||
assert_yolo_summary_bound_to_embedded_training_release,
|
||||
)
|
||||
|
||||
|
||||
def sha256(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
@@ -73,6 +87,11 @@ def main() -> int:
|
||||
parser.add_argument("--internal-val-samples", required=True)
|
||||
parser.add_argument("--version", required=True)
|
||||
parser.add_argument("--output-dir", type=Path, required=True)
|
||||
parser.add_argument(
|
||||
"--fixture-mode",
|
||||
action="store_true",
|
||||
help="Accept only an explicitly fixture-only source corpus; never use for operational holdout rotation.",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
role_slugs = {
|
||||
@@ -86,6 +105,20 @@ def main() -> int:
|
||||
raise SystemExit("rotated holdout lists overlap")
|
||||
|
||||
manifest = json.loads(args.corpus_manifest.read_text(encoding="utf-8"))
|
||||
try:
|
||||
assert_frozen_manifest_training_eligible(
|
||||
args.corpus_manifest,
|
||||
fixture_mode=args.fixture_mode,
|
||||
verify_live=True,
|
||||
)
|
||||
for path in args.source_summary:
|
||||
assert_yolo_summary_bound_to_embedded_training_release(
|
||||
summary_path=path,
|
||||
corpus_manifest=args.corpus_manifest,
|
||||
fixture_mode=args.fixture_mode,
|
||||
)
|
||||
except (TrainingEligibilityError, TrainingReleaseError) as exc:
|
||||
raise SystemExit(str(exc)) from exc
|
||||
source_samples = {item["sample_slug"]: item for item in manifest["samples"]}
|
||||
missing = sorted(all_holdouts - source_samples.keys())
|
||||
if missing:
|
||||
@@ -126,6 +159,18 @@ def main() -> int:
|
||||
args.output_dir.mkdir(parents=True, exist_ok=True)
|
||||
manifest_path = args.output_dir / "operator_samples_manifest.json"
|
||||
write_json(manifest_path, rotated_manifest)
|
||||
write_json(
|
||||
args.output_dir / "corpus-freeze.json",
|
||||
{
|
||||
"schema_version": 2,
|
||||
"dataset_version": rotated_manifest["dataset_version"],
|
||||
"manifest_sha256": sha256(manifest_path),
|
||||
"sample_count": len(rotated_manifest["samples"]),
|
||||
"immutable": True,
|
||||
"training_eligibility_policy": rotated_manifest["training_eligibility"]["policy_version"],
|
||||
"fixture_mode": bool(args.fixture_mode),
|
||||
},
|
||||
)
|
||||
leakage = audit_spatial_leakage(rotated_manifest["samples"])
|
||||
write_json(args.output_dir / "spatial-leakage-audit.json", leakage)
|
||||
if leakage["status"] != "ok":
|
||||
@@ -193,6 +238,11 @@ def main() -> int:
|
||||
"fit_samples_in_internal_validation": sorted(
|
||||
{tile["sample_slug"] for tile in train_tiles} & {tile["sample_slug"] for tile in internal_val_tiles}
|
||||
),
|
||||
"fixture_mode": bool(args.fixture_mode),
|
||||
"training_eligible": False,
|
||||
"training_eligibility_reason": (
|
||||
"A rotated split needs a new human review, label contract validation and immutable training release."
|
||||
),
|
||||
}
|
||||
if evidence["protected_samples_in_training"]:
|
||||
raise SystemExit("protected samples leaked into rotated training lists")
|
||||
|
||||
@@ -0,0 +1,133 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Emit reproducible local evidence for the Phase-2 data foundation.
|
||||
|
||||
The collector does not connect to PostgreSQL, source providers, a GPU or any
|
||||
training corpus. It inventories the versioned source-policy and contract
|
||||
definitions that are present in this checkout. Runtime migration/application
|
||||
evidence is recorded separately because it needs an explicitly chosen target
|
||||
database and must never be inferred from this static report.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
from collections import Counter
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
BACKEND = ROOT / "backend"
|
||||
if str(BACKEND) not in sys.path:
|
||||
sys.path.insert(0, str(BACKEND))
|
||||
|
||||
from app.services.data_contract_validation import build_default_data_contract_registry # noqa: E402
|
||||
from app.services.source_registry_service import SERVER_OWNED_SOURCE_DEFINITIONS # noqa: E402
|
||||
|
||||
|
||||
def _git_value(*args: str) -> str | None:
|
||||
try:
|
||||
return subprocess.check_output(
|
||||
["git", *args],
|
||||
cwd=ROOT,
|
||||
text=True,
|
||||
stderr=subprocess.DEVNULL,
|
||||
).strip() or None
|
||||
except (OSError, subprocess.CalledProcessError):
|
||||
return None
|
||||
|
||||
|
||||
def collect() -> dict[str, Any]:
|
||||
definitions = list(SERVER_OWNED_SOURCE_DEFINITIONS.values())
|
||||
contracts = build_default_data_contract_registry().registered_contracts()
|
||||
classification_counts = Counter(item.classification for item in definitions)
|
||||
source_items = [
|
||||
{
|
||||
"source_key": item.source_key,
|
||||
"classification": item.classification,
|
||||
"authority_name": item.authority_name,
|
||||
"authority_scope": item.authority_scope,
|
||||
"provider_adapter_key": item.provider_adapter_key,
|
||||
"default_crs": item.default_crs,
|
||||
"default_units": item.default_units,
|
||||
"freshness_status": item.freshness_status,
|
||||
"ingest_status": item.ingest_status,
|
||||
"ground_truth_allowed": bool((item.usage_policy or {}).get("ground_truth_allowed")),
|
||||
"training_allowed": bool((item.usage_policy or {}).get("training_allowed")),
|
||||
}
|
||||
for item in sorted(definitions, key=lambda definition: definition.source_key)
|
||||
]
|
||||
contract_items = [
|
||||
{
|
||||
"key": item.key,
|
||||
"version": item.version,
|
||||
"kind": item.kind.value,
|
||||
"fingerprint_sha256": item.fingerprint(),
|
||||
"canonical_storage_crs": item.canonical_storage_crs,
|
||||
"accepted_source_crs": sorted(item.accepted_source_crs),
|
||||
"required_metadata_fields": list(item.required_metadata_fields),
|
||||
"requires_source_registry": item.lineage_rules.require_source_registry,
|
||||
"requires_source_snapshot": item.lineage_rules.require_source_snapshot,
|
||||
"requires_upstream_assets": item.lineage_rules.require_upstream_assets,
|
||||
}
|
||||
for item in sorted(contracts, key=lambda contract: (contract.kind.value, contract.key, contract.version))
|
||||
]
|
||||
return {
|
||||
"schema_version": 1,
|
||||
"program": "GeoIntel Accuracy Improvement Program",
|
||||
"phase": "P2",
|
||||
"collected_at": datetime.now(UTC).isoformat(),
|
||||
"repository": {
|
||||
"branch": _git_value("branch", "--show-current"),
|
||||
"head": _git_value("rev-parse", "HEAD"),
|
||||
"dirty": bool(_git_value("status", "--porcelain")),
|
||||
},
|
||||
"scope": "Belgium and the Belgian North Sea",
|
||||
"migration_revision": "202608010001",
|
||||
"source_registry": {
|
||||
"definition_count": len(source_items),
|
||||
"classification_counts": dict(sorted(classification_counts.items())),
|
||||
"required_building_policy": {
|
||||
"grb_primary_building_validation": SERVER_OWNED_SOURCE_DEFINITIONS["grb"].usage_policy[
|
||||
"validation_authority"
|
||||
].get("building_validation"),
|
||||
"buildings_register_classification": SERVER_OWNED_SOURCE_DEFINITIONS[
|
||||
"digitaal_vlaanderen_buildings_addresses_register"
|
||||
].classification,
|
||||
"sentinel_2_classification": SERVER_OWNED_SOURCE_DEFINITIONS["sentinel_2"].classification,
|
||||
"dhmv_classification": SERVER_OWNED_SOURCE_DEFINITIONS["digitaal_vlaanderen_dhmv"].classification,
|
||||
"osm_ground_truth_allowed": SERVER_OWNED_SOURCE_DEFINITIONS["osm"].usage_policy[
|
||||
"ground_truth_allowed"
|
||||
],
|
||||
},
|
||||
"definitions": source_items,
|
||||
},
|
||||
"data_contracts": contract_items,
|
||||
"claim_boundary": (
|
||||
"Static policy/contract inventory only. It does not attest that a database migration ran, "
|
||||
"that legacy rows are complete, or that a model/corpus is release-ready."
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument(
|
||||
"--output",
|
||||
type=Path,
|
||||
default=ROOT / "artifacts" / "evidence" / "accuracy" / "P2" / "source-contract-inventory.json",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
result = collect()
|
||||
args.output.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.output.write_text(json.dumps(result, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
||||
print(json.dumps({"output": str(args.output), "source_count": result["source_registry"]["definition_count"]}))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -13,6 +13,21 @@ from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
SCRIPT_DIR = Path(__file__).resolve().parent
|
||||
if str(SCRIPT_DIR) not in sys.path:
|
||||
sys.path.insert(0, str(SCRIPT_DIR))
|
||||
|
||||
from training_dataset_eligibility import ( # noqa: E402
|
||||
TrainingEligibilityError,
|
||||
assert_frozen_manifest_training_eligible,
|
||||
)
|
||||
from training_release_manifest import ( # noqa: E402
|
||||
TrainingReleaseError,
|
||||
assert_training_release_eligible,
|
||||
human_review_audit_failures,
|
||||
training_release_paths,
|
||||
)
|
||||
|
||||
|
||||
def sha256(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
@@ -32,11 +47,14 @@ def write_json(path: Path, value: dict[str, Any]) -> None:
|
||||
def dataset_audit_failures(
|
||||
audit: dict[str, Any],
|
||||
train_quality_audit: dict[str, Any],
|
||||
*,
|
||||
fixture_mode: bool = False,
|
||||
) -> list[str]:
|
||||
"""Return automated corpus blockers while leaving final human review deferred."""
|
||||
"""Return fail-closed corpus blockers, including human review in normal mode."""
|
||||
failures = [str(item) for item in audit.get("failures") or []]
|
||||
status = audit.get("status")
|
||||
if status not in {"ok", "needs_human_review"}:
|
||||
permitted_statuses = {"ok", "needs_human_review"} if fixture_mode else {"ok"}
|
||||
if status not in permitted_statuses:
|
||||
failures.append(f"unsupported audit status: {status}")
|
||||
if audit.get("manifest_immutable") is not True:
|
||||
failures.append("corpus manifest is not immutable")
|
||||
@@ -50,9 +68,56 @@ def dataset_audit_failures(
|
||||
failures.append("train tile quality audit contains missing label files")
|
||||
if int(train_quality_audit.get("low_variance_positive_tile_count", -1)) != 0:
|
||||
failures.append("dataset contains blank/low-variance positive tiles")
|
||||
if not fixture_mode:
|
||||
failures.extend(human_review_audit_failures(audit))
|
||||
return failures
|
||||
|
||||
|
||||
def verify_training_inputs(
|
||||
*,
|
||||
train_yaml: Path,
|
||||
corpus_manifest: Path,
|
||||
fixture_mode: bool,
|
||||
) -> dict[str, Any]:
|
||||
"""Re-check every immutable input before initial, retry or resume training."""
|
||||
|
||||
try:
|
||||
assert_frozen_manifest_training_eligible(
|
||||
corpus_manifest,
|
||||
fixture_mode=fixture_mode,
|
||||
verify_live=True,
|
||||
)
|
||||
return assert_training_release_eligible(
|
||||
train_yaml=train_yaml,
|
||||
corpus_manifest=corpus_manifest,
|
||||
fixture_mode=fixture_mode,
|
||||
)
|
||||
except (TrainingEligibilityError, TrainingReleaseError) as exc:
|
||||
raise TrainingReleaseError(str(exc)) from exc
|
||||
|
||||
|
||||
def assert_dataset_audit_bound_to_release(
|
||||
*,
|
||||
release: dict[str, Any],
|
||||
dataset_audit: Path,
|
||||
fixture_mode: bool,
|
||||
) -> None:
|
||||
"""Do not let a caller swap the reviewed corpus audit after release sealing."""
|
||||
|
||||
if fixture_mode:
|
||||
return
|
||||
review = release.get("human_review")
|
||||
if not isinstance(review, dict):
|
||||
raise TrainingReleaseError("Training release has no human-review audit binding")
|
||||
review_audit_path = review.get("audit_path")
|
||||
if not isinstance(review_audit_path, str) or not review_audit_path:
|
||||
raise TrainingReleaseError("Training release human-review audit path is missing")
|
||||
if Path(review_audit_path).resolve(strict=False) != dataset_audit.resolve(strict=False):
|
||||
raise TrainingReleaseError(
|
||||
"--dataset-audit does not match the immutable training-release human-review audit"
|
||||
)
|
||||
|
||||
|
||||
def select_calibration_threshold(report: dict[str, Any]) -> dict[str, Any]:
|
||||
"""Choose a threshold without consulting test or background evidence."""
|
||||
eligible = [item for item in report["sweeps"] if item["pure_empty_false_positives"] == 0]
|
||||
@@ -111,6 +176,16 @@ def rejected_candidate_score(assessment: dict[str, Any]) -> tuple[float, ...]:
|
||||
return tuple(normalized)
|
||||
|
||||
|
||||
def protected_feedback_roles(assessment: dict[str, Any]) -> list[str]:
|
||||
"""Return protected evidence roles that make iterative retraining illegal."""
|
||||
|
||||
return [
|
||||
role
|
||||
for role in ("test", "background")
|
||||
if assessment.get(role) is not None
|
||||
]
|
||||
|
||||
|
||||
def training_command(
|
||||
yolo: str,
|
||||
*,
|
||||
@@ -183,17 +258,23 @@ def failure_sampling_command(
|
||||
corpus_manifest: Path,
|
||||
assessment: Path,
|
||||
output_dir: Path,
|
||||
review_audit: Path,
|
||||
sampling_round: int = 0,
|
||||
fixture_mode: bool = False,
|
||||
) -> list[str]:
|
||||
return [
|
||||
command = [
|
||||
sys.executable,
|
||||
str(scripts_dir / "build_failure_driven_yolo_sampling.py"),
|
||||
"--summary", str(train_summary),
|
||||
"--corpus-manifest", str(corpus_manifest),
|
||||
"--assessment", str(assessment),
|
||||
"--output-dir", str(output_dir),
|
||||
"--review-audit", str(review_audit),
|
||||
"--sampling-round", str(sampling_round),
|
||||
]
|
||||
if fixture_mode:
|
||||
command.append("--fixture-mode")
|
||||
return command
|
||||
|
||||
|
||||
def resumable_training_command(yolo: str, checkpoint: Path) -> list[str]:
|
||||
@@ -259,6 +340,14 @@ def main() -> int:
|
||||
parser.add_argument("--min-region-recall", type=float, default=0.4)
|
||||
parser.add_argument("--max-pure-empty-fp", type=int, default=0)
|
||||
parser.add_argument("--dry-run", action="store_true")
|
||||
parser.add_argument(
|
||||
"--fixture-mode",
|
||||
action="store_true",
|
||||
help=(
|
||||
"Accept an explicitly fixture-only corpus manifest. "
|
||||
"This mode is prohibited for operational training."
|
||||
),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--evaluate-initial-model",
|
||||
action="store_true",
|
||||
@@ -267,9 +356,29 @@ def main() -> int:
|
||||
args = parser.parse_args()
|
||||
if args.iterations < 1:
|
||||
raise SystemExit("--iterations must be positive")
|
||||
try:
|
||||
initial_release = verify_training_inputs(
|
||||
train_yaml=args.train_yaml,
|
||||
corpus_manifest=args.corpus_manifest,
|
||||
fixture_mode=args.fixture_mode,
|
||||
)
|
||||
except TrainingReleaseError as exc:
|
||||
raise SystemExit(str(exc)) from exc
|
||||
try:
|
||||
assert_dataset_audit_bound_to_release(
|
||||
release=initial_release,
|
||||
dataset_audit=args.dataset_audit,
|
||||
fixture_mode=args.fixture_mode,
|
||||
)
|
||||
except TrainingReleaseError as exc:
|
||||
raise SystemExit(str(exc)) from exc
|
||||
dataset_audit = json.loads(args.dataset_audit.read_text(encoding="utf-8"))
|
||||
train_quality_audit = json.loads(args.train_quality_audit.read_text(encoding="utf-8"))
|
||||
audit_failures = dataset_audit_failures(dataset_audit, train_quality_audit)
|
||||
audit_failures = dataset_audit_failures(
|
||||
dataset_audit,
|
||||
train_quality_audit,
|
||||
fixture_mode=args.fixture_mode,
|
||||
)
|
||||
if audit_failures:
|
||||
raise SystemExit(f"Dataset audit is not eligible for training: {audit_failures}")
|
||||
|
||||
@@ -285,6 +394,13 @@ def main() -> int:
|
||||
"train_quality_audit": str(args.train_quality_audit),
|
||||
"train_quality_audit_sha256": sha256(args.train_quality_audit),
|
||||
"corpus_manifest": str(args.corpus_manifest),
|
||||
"corpus_manifest_sha256": sha256(args.corpus_manifest),
|
||||
"initial_training_release": str(training_release_paths(args.train_yaml)["release_manifest"]),
|
||||
"initial_training_release_sha256": sha256(
|
||||
training_release_paths(args.train_yaml)["release_manifest"]
|
||||
),
|
||||
"initial_training_release_contract": initial_release["contract_version"],
|
||||
"fixture_mode": bool(args.fixture_mode),
|
||||
"iterations": [],
|
||||
}
|
||||
if state_path.is_file():
|
||||
@@ -304,6 +420,26 @@ def main() -> int:
|
||||
evaluate_existing = args.evaluate_initial_model and offset == 0 and not state["iterations"]
|
||||
partial_checkpoint = train_run / "weights" / "last.pt"
|
||||
resume_partial = not evaluate_existing and partial_checkpoint.is_file()
|
||||
try:
|
||||
release = verify_training_inputs(
|
||||
train_yaml=train_yaml,
|
||||
corpus_manifest=args.corpus_manifest,
|
||||
fixture_mode=args.fixture_mode,
|
||||
)
|
||||
except TrainingReleaseError as exc:
|
||||
raise RuntimeError(
|
||||
f"Training inputs changed before {name}; refusing initial/retry/resume execution: {exc}"
|
||||
) from exc
|
||||
try:
|
||||
assert_dataset_audit_bound_to_release(
|
||||
release=release,
|
||||
dataset_audit=args.dataset_audit,
|
||||
fixture_mode=args.fixture_mode,
|
||||
)
|
||||
except TrainingReleaseError as exc:
|
||||
raise RuntimeError(
|
||||
f"Training audit changed before {name}; refusing initial/retry/resume execution: {exc}"
|
||||
) from exc
|
||||
command = None if evaluate_existing else (
|
||||
resumable_training_command(args.yolo, partial_checkpoint)
|
||||
if resume_partial else training_command(
|
||||
@@ -454,11 +590,27 @@ def main() -> int:
|
||||
"candidate_sha256": sha256(candidate),
|
||||
"training_skipped_for_existing_checkpoint": evaluate_existing,
|
||||
"training_resumed_from_partial_checkpoint": resume_partial,
|
||||
"training_release": str(training_release_paths(train_yaml)["release_manifest"]),
|
||||
"training_release_sha256": sha256(training_release_paths(train_yaml)["release_manifest"]),
|
||||
"training_release_asset_manifest_sha256": release["asset_manifest"]["sha256"],
|
||||
"assessment": str(assessment),
|
||||
"status": decision["status"],
|
||||
"failures": decision["failures"],
|
||||
}
|
||||
state["iterations"].append(record)
|
||||
protected_feedback = protected_feedback_roles(decision)
|
||||
if decision["status"] != "training_complete" and protected_feedback:
|
||||
record["protected_feedback_blocked"] = protected_feedback
|
||||
record["retraining_prohibited"] = True
|
||||
state["status"] = "protected_evaluation_rejected"
|
||||
state["stopped_at"] = datetime.now(UTC).isoformat()
|
||||
state["stop_reason"] = (
|
||||
"Protected test/background evidence was opened for a rejected candidate; "
|
||||
"its results cannot generate another training YAML."
|
||||
)
|
||||
write_json(state_path, state)
|
||||
print(json.dumps(state, indent=2))
|
||||
return 3
|
||||
score = rejected_candidate_score(decision) if decision["status"] != "training_complete" else ()
|
||||
incumbent_score = tuple(state.get("incumbent_rejected_score", ()))
|
||||
if not incumbent_score or score > incumbent_score:
|
||||
@@ -482,14 +634,29 @@ def main() -> int:
|
||||
corpus_manifest=args.corpus_manifest,
|
||||
assessment=assessment,
|
||||
output_dir=sampling_dir,
|
||||
review_audit=args.dataset_audit,
|
||||
sampling_round=index,
|
||||
fixture_mode=args.fixture_mode,
|
||||
),
|
||||
iteration_dir / "failure-driven-sampling.log",
|
||||
)
|
||||
sampling_evidence = sampling_dir / "failure-driven-sampling.json"
|
||||
next_train_yaml = sampling_dir / "dataset.yaml"
|
||||
if not sampling_evidence.is_file() or not next_train_yaml.is_file():
|
||||
next_release_paths = training_release_paths(next_train_yaml)
|
||||
if not sampling_evidence.is_file() or not next_train_yaml.is_file() or not all(
|
||||
path.is_file() for path in next_release_paths.values()
|
||||
):
|
||||
raise RuntimeError("Failure-driven sampling produced incomplete evidence")
|
||||
try:
|
||||
verify_training_inputs(
|
||||
train_yaml=next_train_yaml,
|
||||
corpus_manifest=args.corpus_manifest,
|
||||
fixture_mode=args.fixture_mode,
|
||||
)
|
||||
except TrainingReleaseError as exc:
|
||||
raise RuntimeError(
|
||||
f"Failure-driven sampling produced an unbound training release: {exc}"
|
||||
) from exc
|
||||
record["failure_driven_sampling"] = str(sampling_evidence)
|
||||
record["failure_driven_sampling_sha256"] = sha256(sampling_evidence)
|
||||
record["next_train_yaml"] = str(next_train_yaml)
|
||||
|
||||
@@ -3,6 +3,8 @@ from __future__ import annotations
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
from datetime import UTC, datetime
|
||||
from hashlib import sha256
|
||||
from pathlib import Path
|
||||
from uuid import uuid4
|
||||
|
||||
@@ -12,9 +14,10 @@ BACKEND_ROOT = ROOT / "backend"
|
||||
if str(BACKEND_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(BACKEND_ROOT))
|
||||
|
||||
from app.models import Area, Dataset, Metric, QualityCheck # noqa: E402
|
||||
from app.models import Dataset, Metric, QualityCheck, SourceRegistry, SourceSnapshot # noqa: E402
|
||||
from app.services.qa_service import QaService # noqa: E402
|
||||
from app.services.quality_service import QualityService # noqa: E402
|
||||
from app.services.source_registry_service import SourceRegistryService # noqa: E402
|
||||
|
||||
|
||||
class BenchmarkSession:
|
||||
@@ -55,18 +58,58 @@ def _load_manifest() -> dict:
|
||||
|
||||
|
||||
def _dataset(dataset_id, project_id, name: str, path: Path, *, role: str) -> Dataset:
|
||||
# The benchmark is still synthetic evidence, but it intentionally models
|
||||
# the same complete Phase 2 provenance binding required at the QA service
|
||||
# boundary. The candidate is a derived fixture; the reference simulates
|
||||
# a governed GRB building snapshot. No legacy fixture exception is used.
|
||||
source_key = "grb" if role == "reference" else "derived"
|
||||
source = SourceRegistry(
|
||||
id=uuid4(),
|
||||
**SourceRegistryService.definition_for(source_key).as_model_values(),
|
||||
)
|
||||
checksum = sha256(path.read_bytes()).hexdigest()
|
||||
snapshot = SourceSnapshot(
|
||||
id=uuid4(),
|
||||
source_registry_id=source.id,
|
||||
snapshot_key=f"golden-qa:{source_key}:{checksum}",
|
||||
checksum_sha256=checksum,
|
||||
fetched_at=datetime.now(UTC),
|
||||
crs="EPSG:4326",
|
||||
units=source.default_units,
|
||||
spatial_resolution_json={"status": "fixture"},
|
||||
temporal_coverage_json={"status": "fixture"},
|
||||
geographic_coverage_json={"scope": "golden-qa-fixture"},
|
||||
observed_schema_json={"dataset_type": "vector", "fixture_mode": True},
|
||||
freshness_status="current",
|
||||
ingest_status="ingested",
|
||||
known_limitations_json=["Synthetic golden benchmark fixture; not production evidence."],
|
||||
snapshot_metadata_json={"fixture_mode": True, "benchmark": "golden-qa"},
|
||||
)
|
||||
return Dataset(
|
||||
id=dataset_id,
|
||||
project_id=project_id,
|
||||
name=name,
|
||||
dataset_type="vector",
|
||||
source="golden_fixture",
|
||||
dataset_role=role,
|
||||
source_name="fixture",
|
||||
source="governed golden benchmark fixture",
|
||||
dataset_role="reference" if role == "reference" else "derived",
|
||||
source_name=source.source_key,
|
||||
reference_layer_name="buildings" if role == "reference" else None,
|
||||
storage_path=str(path),
|
||||
crs="EPSG:4326",
|
||||
metadata_json={"crs_assumed": False},
|
||||
checksum_sha256=checksum,
|
||||
source_registry_id=source.id,
|
||||
source_snapshot_id=snapshot.id,
|
||||
source_registry=source,
|
||||
source_snapshot=snapshot,
|
||||
data_contract_key="geointel.vector.geojson",
|
||||
data_contract_version="1.0.0",
|
||||
validation_status="passed",
|
||||
provenance_status="complete",
|
||||
lineage_status="not_applicable",
|
||||
quarantine_status="not_quarantined",
|
||||
metadata_json={"crs_assumed": False, "fixture_mode": True},
|
||||
source_metadata={"fixture_mode": True, "source_registry_key": source.source_key},
|
||||
provenance_metadata={"fixture_mode": True, "source_snapshot_id": str(snapshot.id)},
|
||||
status="ready",
|
||||
)
|
||||
|
||||
|
||||
@@ -109,6 +109,10 @@ ${PYTHON_BIN} -m py_compile scripts/audit_source_freshness.py
|
||||
${PYTHON_BIN} -m py_compile scripts/manage_grb_refresh.py
|
||||
${PYTHON_BIN} -m py_compile scripts/export_operator_yolo_dataset.py
|
||||
${PYTHON_BIN} -m py_compile scripts/export_operator_yolo_tile_dataset.py
|
||||
${PYTHON_BIN} -m py_compile scripts/training_dataset_eligibility.py
|
||||
${PYTHON_BIN} -m py_compile scripts/training_release_manifest.py
|
||||
${PYTHON_BIN} -m py_compile scripts/run_belgium_building_training_loop.py
|
||||
${PYTHON_BIN} -m py_compile scripts/build_failure_driven_yolo_sampling.py
|
||||
${PYTHON_BIN} -m py_compile scripts/audit_operator_yolo_dataset_quality.py
|
||||
${PYTHON_BIN} -m py_compile scripts/build_detection_model_promotion_report.py
|
||||
${PYTHON_BIN} -m py_compile scripts/build_mol_operational_benchmark_report.py
|
||||
|
||||
@@ -6,10 +6,20 @@ from __future__ import annotations
|
||||
import argparse
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
|
||||
SCRIPT_DIR = Path(__file__).resolve().parent
|
||||
if str(SCRIPT_DIR) not in sys.path:
|
||||
sys.path.insert(0, str(SCRIPT_DIR))
|
||||
|
||||
from training_release_manifest import ( # noqa: E402
|
||||
TrainingReleaseError,
|
||||
assert_training_release_eligible,
|
||||
)
|
||||
|
||||
|
||||
def container_running(container: str) -> bool:
|
||||
result = subprocess.run(
|
||||
@@ -44,11 +54,33 @@ def load_completion_command(path: Path) -> list[str]:
|
||||
return command
|
||||
|
||||
|
||||
def verify_training_release(
|
||||
*,
|
||||
train_yaml: Path,
|
||||
corpus_manifest: Path,
|
||||
fixture_mode: bool,
|
||||
) -> None:
|
||||
"""Re-check the exact release before each detached resume command."""
|
||||
|
||||
assert_training_release_eligible(
|
||||
train_yaml=train_yaml,
|
||||
corpus_manifest=corpus_manifest,
|
||||
fixture_mode=fixture_mode,
|
||||
)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--container", required=True)
|
||||
parser.add_argument("--host-run-dir", type=Path, required=True)
|
||||
parser.add_argument("--container-checkpoint", required=True)
|
||||
parser.add_argument("--train-yaml", type=Path, required=True)
|
||||
parser.add_argument("--corpus-manifest", type=Path, required=True)
|
||||
parser.add_argument(
|
||||
"--fixture-mode",
|
||||
action="store_true",
|
||||
help="Only valid for an explicitly fixture-only frozen corpus release.",
|
||||
)
|
||||
parser.add_argument("--run-marker", required=True)
|
||||
parser.add_argument("--yolo", default="/opt/geointel/venv/bin/yolo")
|
||||
parser.add_argument("--poll-seconds", type=int, default=30)
|
||||
@@ -101,6 +133,17 @@ def main() -> int:
|
||||
state["status"] = "resume_budget_exhausted"
|
||||
write_state(state_path, state)
|
||||
return 3
|
||||
try:
|
||||
verify_training_release(
|
||||
train_yaml=args.train_yaml,
|
||||
corpus_manifest=args.corpus_manifest,
|
||||
fixture_mode=args.fixture_mode,
|
||||
)
|
||||
except TrainingReleaseError as exc:
|
||||
state["status"] = "training_release_verification_failed"
|
||||
state["training_release_error"] = str(exc)
|
||||
write_state(state_path, state)
|
||||
return 6
|
||||
result = subprocess.run(
|
||||
["docker", "exec", "-d", args.container, args.yolo, "train",
|
||||
f"resume={args.container_checkpoint}", "device=0"],
|
||||
|
||||
@@ -4,8 +4,248 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
import sys
|
||||
from collections import Counter
|
||||
from pathlib import Path, PurePosixPath
|
||||
from typing import Any, Mapping
|
||||
|
||||
|
||||
SCRIPT_DIR = Path(__file__).resolve().parent
|
||||
if str(SCRIPT_DIR) not in sys.path:
|
||||
sys.path.insert(0, str(SCRIPT_DIR))
|
||||
|
||||
from training_dataset_eligibility import ( # noqa: E402
|
||||
TrainingEligibilityError,
|
||||
assert_frozen_manifest_training_eligible,
|
||||
)
|
||||
from training_release_manifest import ( # noqa: E402
|
||||
TrainingReleaseError,
|
||||
assert_yolo_summary_bound_to_embedded_training_release,
|
||||
)
|
||||
|
||||
|
||||
PROPOSAL_DATASET_PROVENANCE_NAME = "proposal-dataset-provenance.json"
|
||||
PROPOSAL_DATASET_SCHEMA_VERSION = 1
|
||||
_CROP_IMAGE_SUFFIXES = {".bmp", ".jpeg", ".jpg", ".png", ".tif", ".tiff", ".webp"}
|
||||
|
||||
|
||||
class ProposalDatasetProvenanceError(ValueError):
|
||||
"""Raised when proposal crops no longer match their governed provenance."""
|
||||
|
||||
|
||||
def sha256(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as stream:
|
||||
for chunk in iter(lambda: stream.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def _canonical_json_bytes(payload: Mapping[str, Any]) -> bytes:
|
||||
return json.dumps(payload, ensure_ascii=False, separators=(",", ":"), sort_keys=True).encode("utf-8")
|
||||
|
||||
|
||||
def _payload_sha256(payload: Mapping[str, Any]) -> str:
|
||||
normalized = dict(payload)
|
||||
normalized.pop("manifest_sha256", None)
|
||||
return hashlib.sha256(_canonical_json_bytes(normalized)).hexdigest()
|
||||
|
||||
|
||||
def _is_sha256(value: Any) -> bool:
|
||||
return isinstance(value, str) and len(value) == 64 and all(character in "0123456789abcdef" for character in value)
|
||||
|
||||
|
||||
def _require_mapping(value: Any, *, field: str) -> Mapping[str, Any]:
|
||||
if not isinstance(value, Mapping):
|
||||
raise ProposalDatasetProvenanceError(f"proposal dataset provenance {field} must be an object")
|
||||
return value
|
||||
|
||||
|
||||
def _require_sha256(value: Any, *, field: str) -> str:
|
||||
if not _is_sha256(value):
|
||||
raise ProposalDatasetProvenanceError(f"proposal dataset provenance {field} must be a lowercase SHA-256")
|
||||
return str(value)
|
||||
|
||||
|
||||
def _require_nonempty_path(value: Any, *, field: str) -> str:
|
||||
if not isinstance(value, str) or not value.strip():
|
||||
raise ProposalDatasetProvenanceError(f"proposal dataset provenance {field} must be a non-empty path")
|
||||
return value
|
||||
|
||||
|
||||
def _safe_relative_crop_path(value: Any, *, dataset_root: Path) -> tuple[str, Path]:
|
||||
if not isinstance(value, str) or not value or "\\" in value:
|
||||
raise ProposalDatasetProvenanceError("proposal crop relative_path must be a non-empty POSIX relative path")
|
||||
relative = PurePosixPath(value)
|
||||
if relative.is_absolute() or ".." in relative.parts or "." in relative.parts:
|
||||
raise ProposalDatasetProvenanceError(f"proposal crop has an unsafe relative path: {value!r}")
|
||||
path = (dataset_root / Path(*relative.parts)).resolve(strict=False)
|
||||
try:
|
||||
path.relative_to(dataset_root)
|
||||
except ValueError as exc: # pragma: no cover - resolved-path defence
|
||||
raise ProposalDatasetProvenanceError(f"proposal crop escapes dataset directory: {value!r}") from exc
|
||||
return relative.as_posix(), path
|
||||
|
||||
|
||||
def load_governed_corpus_manifest(corpus_manifest_path: Path, *, fixture_mode: bool) -> dict[str, Any]:
|
||||
"""Assert frozen evidence and current live Dataset eligibility before PyTorch."""
|
||||
|
||||
try:
|
||||
manifest = assert_frozen_manifest_training_eligible(
|
||||
corpus_manifest_path,
|
||||
fixture_mode=fixture_mode,
|
||||
verify_live=True,
|
||||
)
|
||||
except TrainingEligibilityError as exc:
|
||||
raise ProposalDatasetProvenanceError(str(exc)) from exc
|
||||
if not isinstance(manifest, dict): # pragma: no cover - defensive contract boundary
|
||||
raise ProposalDatasetProvenanceError("governed corpus manifest must be a JSON object")
|
||||
return manifest
|
||||
|
||||
|
||||
def validate_proposal_dataset_provenance(
|
||||
dataset_dir: Path,
|
||||
corpus_manifest_path: Path,
|
||||
*,
|
||||
fixture_mode: bool,
|
||||
) -> dict[str, Any]:
|
||||
"""Fail closed unless every trainable crop still matches its source evidence.
|
||||
|
||||
ImageFolder discovers files from the directory tree. Merely verifying a
|
||||
manifest is insufficient: any unbound image placed under train/ or val/
|
||||
would otherwise enter PyTorch training. This validator compares the
|
||||
complete tree with the immutable crop manifest and re-binds it to the
|
||||
supplied frozen corpus and summary.
|
||||
"""
|
||||
|
||||
root = dataset_dir.expanduser().resolve(strict=False)
|
||||
if not root.is_dir():
|
||||
raise ProposalDatasetProvenanceError(f"proposal dataset directory is unavailable: {root}")
|
||||
manifest_path = root / PROPOSAL_DATASET_PROVENANCE_NAME
|
||||
try:
|
||||
payload = json.loads(manifest_path.read_text(encoding="utf-8"))
|
||||
except (OSError, json.JSONDecodeError) as exc:
|
||||
raise ProposalDatasetProvenanceError(f"proposal dataset provenance is unreadable: {manifest_path}") from exc
|
||||
if not isinstance(payload, dict):
|
||||
raise ProposalDatasetProvenanceError("proposal dataset provenance must be a JSON object")
|
||||
if payload.get("schema_version") != PROPOSAL_DATASET_SCHEMA_VERSION:
|
||||
raise ProposalDatasetProvenanceError("proposal dataset provenance schema_version is unsupported")
|
||||
if payload.get("status") != "ok" or payload.get("immutable") is not True:
|
||||
raise ProposalDatasetProvenanceError("proposal dataset provenance is not an immutable successful release")
|
||||
if payload.get("governed_corpus_live_recheck") is not True:
|
||||
raise ProposalDatasetProvenanceError("proposal dataset provenance lacks the governed live-source recheck")
|
||||
if bool(payload.get("fixture_mode")) != fixture_mode:
|
||||
raise ProposalDatasetProvenanceError("proposal dataset provenance fixture-mode binding mismatches this invocation")
|
||||
if payload.get("manifest_sha256") != _payload_sha256(payload):
|
||||
raise ProposalDatasetProvenanceError("proposal dataset provenance manifest checksum mismatches its content")
|
||||
|
||||
expected_corpus_sha256 = sha256(corpus_manifest_path)
|
||||
source = _require_mapping(payload.get("source"), field="source")
|
||||
recorded_corpus = _require_mapping(source.get("corpus_manifest"), field="source.corpus_manifest")
|
||||
if _require_sha256(recorded_corpus.get("sha256"), field="source.corpus_manifest.sha256") != expected_corpus_sha256:
|
||||
raise ProposalDatasetProvenanceError("proposal crops are not bound to the supplied governed corpus manifest")
|
||||
|
||||
corpus_freeze_path = corpus_manifest_path.parent / "corpus-freeze.json"
|
||||
recorded_freeze = _require_mapping(source.get("corpus_freeze"), field="source.corpus_freeze")
|
||||
if not corpus_freeze_path.is_file() or _require_sha256(
|
||||
recorded_freeze.get("sha256"), field="source.corpus_freeze.sha256"
|
||||
) != sha256(corpus_freeze_path):
|
||||
raise ProposalDatasetProvenanceError("proposal crops are not bound to the current frozen corpus sidecar")
|
||||
|
||||
summary = _require_mapping(source.get("summary"), field="source.summary")
|
||||
summary_path_value = _require_nonempty_path(summary.get("path"), field="source.summary.path")
|
||||
summary_path = Path(summary_path_value).expanduser().resolve(strict=False)
|
||||
if not summary_path.is_file():
|
||||
raise ProposalDatasetProvenanceError(f"proposal source summary is unavailable: {summary_path}")
|
||||
if _require_sha256(summary.get("sha256"), field="source.summary.sha256") != sha256(summary_path):
|
||||
raise ProposalDatasetProvenanceError("proposal source summary checksum no longer matches the crop provenance")
|
||||
try:
|
||||
summary_payload = json.loads(summary_path.read_text(encoding="utf-8"))
|
||||
except (OSError, json.JSONDecodeError) as exc:
|
||||
raise ProposalDatasetProvenanceError(f"proposal source summary is unreadable: {summary_path}") from exc
|
||||
if not isinstance(summary_payload, Mapping) or summary_payload.get("source_manifest_sha256") != expected_corpus_sha256:
|
||||
raise ProposalDatasetProvenanceError("proposal source summary is not bound to the supplied governed corpus manifest")
|
||||
if summary.get("source_manifest_sha256") != expected_corpus_sha256:
|
||||
raise ProposalDatasetProvenanceError("proposal crop provenance summary binding is invalid")
|
||||
try:
|
||||
source_release = assert_yolo_summary_bound_to_embedded_training_release(
|
||||
summary_path=summary_path,
|
||||
corpus_manifest=corpus_manifest_path,
|
||||
fixture_mode=fixture_mode,
|
||||
)
|
||||
except TrainingReleaseError as exc:
|
||||
raise ProposalDatasetProvenanceError(
|
||||
"proposal source summary has no eligible immutable training release"
|
||||
) from exc
|
||||
recorded_release = _require_mapping(source.get("training_release"), field="source.training_release")
|
||||
if (
|
||||
recorded_release.get("dataset_yaml_path") != source_release["dataset_yaml"]["path"]
|
||||
or recorded_release.get("dataset_yaml_sha256") != source_release["dataset_yaml"]["sha256"]
|
||||
or recorded_release.get("corpus_manifest_sha256") != source_release["corpus"]["manifest_sha256"]
|
||||
):
|
||||
raise ProposalDatasetProvenanceError("proposal crop provenance release binding is invalid")
|
||||
|
||||
proposal_model = _require_mapping(source.get("proposal_model"), field="source.proposal_model")
|
||||
_require_nonempty_path(proposal_model.get("path"), field="source.proposal_model.path")
|
||||
_require_sha256(proposal_model.get("sha256"), field="source.proposal_model.sha256")
|
||||
entries = payload.get("crops")
|
||||
if not isinstance(entries, list) or not entries:
|
||||
raise ProposalDatasetProvenanceError("proposal dataset provenance has no crop entries")
|
||||
|
||||
expected_paths: set[str] = set()
|
||||
calculated_counts: Counter[str] = Counter()
|
||||
normalized_entries: list[dict[str, Any]] = []
|
||||
for entry in entries:
|
||||
item = _require_mapping(entry, field="crops[]")
|
||||
relative_path, crop_path = _safe_relative_crop_path(item.get("relative_path"), dataset_root=root)
|
||||
if relative_path in expected_paths:
|
||||
raise ProposalDatasetProvenanceError(f"proposal crop provenance has duplicate path: {relative_path}")
|
||||
expected_paths.add(relative_path)
|
||||
split = item.get("split")
|
||||
label = item.get("label")
|
||||
if split not in {"train", "val"} or label not in {"negative", "positive"}:
|
||||
raise ProposalDatasetProvenanceError(f"proposal crop has invalid split/class: {relative_path}")
|
||||
if not isinstance(item.get("sample_slug"), str) or not item["sample_slug"].strip():
|
||||
raise ProposalDatasetProvenanceError(f"proposal crop has no sample identity: {relative_path}")
|
||||
if not crop_path.is_file():
|
||||
raise ProposalDatasetProvenanceError(f"proposal crop is missing: {relative_path}")
|
||||
if _require_sha256(item.get("sha256"), field=f"crops[{relative_path}].sha256") != sha256(crop_path):
|
||||
raise ProposalDatasetProvenanceError(f"proposal crop checksum mismatches provenance: {relative_path}")
|
||||
if item.get("size_bytes") != crop_path.stat().st_size:
|
||||
raise ProposalDatasetProvenanceError(f"proposal crop byte size mismatches provenance: {relative_path}")
|
||||
_require_nonempty_path(item.get("source_image_path"), field=f"crops[{relative_path}].source_image_path")
|
||||
_require_sha256(item.get("source_image_sha256"), field=f"crops[{relative_path}].source_image_sha256")
|
||||
_require_nonempty_path(item.get("source_label_path"), field=f"crops[{relative_path}].source_label_path")
|
||||
_require_sha256(item.get("source_label_sha256"), field=f"crops[{relative_path}].source_label_sha256")
|
||||
calculated_counts[f"{split}/{label}"] += 1
|
||||
normalized_entries.append(dict(item))
|
||||
|
||||
canonical_entries = sorted(normalized_entries, key=lambda item: str(item["relative_path"]))
|
||||
expected_crops_sha256 = hashlib.sha256(_canonical_json_bytes({"crops": canonical_entries})).hexdigest()
|
||||
if payload.get("crops_sha256") != expected_crops_sha256:
|
||||
raise ProposalDatasetProvenanceError("proposal crop collection checksum mismatches provenance")
|
||||
if payload.get("crop_count") != len(entries):
|
||||
raise ProposalDatasetProvenanceError("proposal crop count mismatches provenance")
|
||||
if payload.get("counts") != dict(sorted(calculated_counts.items())):
|
||||
raise ProposalDatasetProvenanceError("proposal crop class counts mismatch provenance")
|
||||
if any(calculated_counts[f"{split}/{label}"] < 1 for split in ("train", "val") for label in ("negative", "positive")):
|
||||
raise ProposalDatasetProvenanceError("proposal dataset has an empty train/validation class")
|
||||
|
||||
actual_paths = {
|
||||
file_path.relative_to(root).as_posix()
|
||||
for file_path in root.rglob("*")
|
||||
if file_path.is_file() and file_path.suffix.lower() in _CROP_IMAGE_SUFFIXES
|
||||
}
|
||||
if actual_paths != expected_paths:
|
||||
unexpected = sorted(actual_paths - expected_paths)
|
||||
missing = sorted(expected_paths - actual_paths)
|
||||
raise ProposalDatasetProvenanceError(
|
||||
"proposal dataset files are not exactly the immutable crop manifest "
|
||||
f"(unexpected={unexpected}, missing={missing})"
|
||||
)
|
||||
return payload
|
||||
|
||||
|
||||
def binary_metrics(scores: list[float], labels: list[int], threshold: float = 0.5) -> dict[str, float | int]:
|
||||
@@ -21,16 +261,44 @@ def binary_metrics(scores: list[float], labels: list[int], threshold: float = 0.
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--dataset-dir", type=Path, required=True)
|
||||
parser.add_argument(
|
||||
"--corpus-manifest",
|
||||
type=Path,
|
||||
required=True,
|
||||
help="The exact frozen governed corpus manifest that produced the proposal crops.",
|
||||
)
|
||||
parser.add_argument("--output-dir", type=Path, required=True)
|
||||
parser.add_argument("--epochs", type=int, default=12)
|
||||
parser.add_argument("--batch", type=int, default=64)
|
||||
parser.add_argument("--lr", type=float, default=1e-4)
|
||||
parser.add_argument("--device", default="cuda:0")
|
||||
parser.add_argument("--export-existing-best", action="store_true")
|
||||
parser.add_argument(
|
||||
"--fixture-mode",
|
||||
action="store_true",
|
||||
help="Permit only an explicitly fixture-only corpus; operational runs always resolve live dataset evidence.",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
if args.output_dir.exists() and not args.export_existing_best:
|
||||
parser.error(f"output already exists: {args.output_dir}")
|
||||
|
||||
# All governed-source checks run before importing torch/torchvision or
|
||||
# touching CUDA. A source revocation therefore cannot start PyTorch work.
|
||||
try:
|
||||
governed_corpus = load_governed_corpus_manifest(
|
||||
args.corpus_manifest,
|
||||
fixture_mode=args.fixture_mode,
|
||||
)
|
||||
proposal_provenance = validate_proposal_dataset_provenance(
|
||||
args.dataset_dir,
|
||||
args.corpus_manifest,
|
||||
fixture_mode=args.fixture_mode,
|
||||
)
|
||||
except ProposalDatasetProvenanceError as exc:
|
||||
raise SystemExit(str(exc)) from exc
|
||||
proposal_provenance_path = args.dataset_dir / PROPOSAL_DATASET_PROVENANCE_NAME
|
||||
proposal_provenance_sha256 = sha256(proposal_provenance_path)
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
from torch.utils.data import DataLoader
|
||||
@@ -54,7 +322,18 @@ def main() -> int:
|
||||
model.load_state_dict(torch.load(state_path, map_location=device))
|
||||
model.eval()
|
||||
torch.jit.script(model).save(str(args.output_dir / "proposal-classifier.torchscript.pt"))
|
||||
print(json.dumps({"status": "exported_existing_best", "model": str(state_path)}))
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"status": "exported_existing_best",
|
||||
"model": str(state_path),
|
||||
"proposal_dataset_provenance": str(proposal_provenance_path),
|
||||
"proposal_dataset_provenance_sha256": proposal_provenance_sha256,
|
||||
"governed_corpus_manifest_sha256": sha256(args.corpus_manifest),
|
||||
"governed_corpus_live_recheck": True,
|
||||
}
|
||||
)
|
||||
)
|
||||
return 0
|
||||
train_loader = DataLoader(train_ds, batch_size=args.batch, shuffle=True, num_workers=0)
|
||||
val_loader = DataLoader(val_ds, batch_size=args.batch, shuffle=False, num_workers=0)
|
||||
@@ -93,11 +372,27 @@ def main() -> int:
|
||||
model.load_state_dict(torch.load(args.output_dir / "best-state.pt", map_location=device))
|
||||
model.eval()
|
||||
scripted = torch.jit.script(model)
|
||||
scripted.save(str(args.output_dir / "proposal-classifier.torchscript.pt"))
|
||||
report = {"schema_version": 1, "status": "ok", "classes": train_ds.class_to_idx,
|
||||
"train_count": len(train_ds), "validation_count": len(val_ds),
|
||||
"best_validation_f1": best_f1, "history": history,
|
||||
"model": str(args.output_dir / "proposal-classifier.torchscript.pt")}
|
||||
model_path = args.output_dir / "proposal-classifier.torchscript.pt"
|
||||
scripted.save(str(model_path))
|
||||
report = {
|
||||
"schema_version": 1,
|
||||
"status": "ok",
|
||||
"classes": train_ds.class_to_idx,
|
||||
"train_count": len(train_ds),
|
||||
"validation_count": len(val_ds),
|
||||
"best_validation_f1": best_f1,
|
||||
"history": history,
|
||||
"model": str(model_path),
|
||||
"model_sha256": sha256(model_path),
|
||||
"proposal_dataset_provenance": str(proposal_provenance_path),
|
||||
"proposal_dataset_provenance_sha256": proposal_provenance_sha256,
|
||||
"proposal_dataset_manifest_sha256": proposal_provenance["manifest_sha256"],
|
||||
"corpus_manifest": str(args.corpus_manifest),
|
||||
"corpus_manifest_sha256": sha256(args.corpus_manifest),
|
||||
"fixture_mode": bool(args.fixture_mode),
|
||||
"governed_corpus_live_recheck": True,
|
||||
"governed_corpus_sample_count": len(governed_corpus.get("samples", [])),
|
||||
}
|
||||
(args.output_dir / "training-report.json").write_text(json.dumps(report, indent=2), encoding="utf-8")
|
||||
return 0
|
||||
|
||||
|
||||
@@ -8,9 +8,23 @@ import hashlib
|
||||
import json
|
||||
import random
|
||||
import shutil
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any, TypeVar
|
||||
|
||||
SCRIPT_DIR = Path(__file__).resolve().parent
|
||||
if str(SCRIPT_DIR) not in sys.path:
|
||||
sys.path.insert(0, str(SCRIPT_DIR))
|
||||
|
||||
from training_dataset_eligibility import ( # noqa: E402
|
||||
TrainingEligibilityError,
|
||||
assert_frozen_manifest_training_eligible,
|
||||
)
|
||||
from training_release_manifest import ( # noqa: E402
|
||||
TrainingReleaseError,
|
||||
assert_yolo_summary_bound_to_training_release,
|
||||
)
|
||||
|
||||
|
||||
T = TypeVar("T")
|
||||
|
||||
@@ -111,6 +125,18 @@ def choose_threshold(probabilities: list[float], labels: list[int]) -> tuple[flo
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--summary", type=Path, required=True)
|
||||
parser.add_argument(
|
||||
"--corpus-manifest",
|
||||
type=Path,
|
||||
required=True,
|
||||
help="Immutable governed corpus manifest that produced the tile summary.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--train-yaml",
|
||||
type=Path,
|
||||
required=True,
|
||||
help="YOLO dataset YAML with an eligible immutable GeoIntel training release.",
|
||||
)
|
||||
parser.add_argument("--proposal-model", type=Path, required=True)
|
||||
parser.add_argument("--output-dir", type=Path, required=True)
|
||||
parser.add_argument("--device", default="cuda:0")
|
||||
@@ -124,7 +150,31 @@ def main() -> int:
|
||||
parser.add_argument("--proposal-chunk-size", type=int, default=16)
|
||||
parser.add_argument("--seed", type=int, default=20260727)
|
||||
parser.add_argument("--force", action="store_true")
|
||||
parser.add_argument(
|
||||
"--fixture-mode",
|
||||
action="store_true",
|
||||
help="Accept only an explicitly fixture-only corpus manifest; never use for operational training.",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
try:
|
||||
assert_frozen_manifest_training_eligible(
|
||||
args.corpus_manifest,
|
||||
fixture_mode=args.fixture_mode,
|
||||
verify_live=True,
|
||||
)
|
||||
assert_yolo_summary_bound_to_training_release(
|
||||
summary_path=args.summary,
|
||||
train_yaml=args.train_yaml,
|
||||
corpus_manifest=args.corpus_manifest,
|
||||
fixture_mode=args.fixture_mode,
|
||||
)
|
||||
except (TrainingEligibilityError, TrainingReleaseError) as exc:
|
||||
raise SystemExit(str(exc)) from exc
|
||||
summary = json.loads(args.summary.read_text(encoding="utf-8"))
|
||||
if summary.get("source_manifest_sha256") != sha256(args.corpus_manifest):
|
||||
raise SystemExit(
|
||||
"Proposal-filter tile summary is not bound to the supplied governed corpus manifest"
|
||||
)
|
||||
if args.output_dir.exists():
|
||||
if not args.force:
|
||||
raise SystemExit(f"Output exists: {args.output_dir}")
|
||||
@@ -133,12 +183,11 @@ def main() -> int:
|
||||
import torch
|
||||
from torch import nn
|
||||
from torch.utils.data import DataLoader
|
||||
from torchvision import datasets, transforms
|
||||
from torchvision import datasets
|
||||
from torchvision.models import ResNet18_Weights, resnet18
|
||||
from ultralytics import YOLO
|
||||
|
||||
torch.manual_seed(args.seed)
|
||||
summary = json.loads(args.summary.read_text(encoding="utf-8"))
|
||||
split_tiles = {
|
||||
split: [tile for tile in summary["tiles"] if tile.get("kept", True) and tile["split"] == split]
|
||||
for split in ("train", "val")
|
||||
@@ -214,6 +263,9 @@ def main() -> int:
|
||||
evidence = {
|
||||
"schema_version": 1, "status": "ok", "architecture": "resnet18_binary_proposal_filter",
|
||||
"summary": str(args.summary), "summary_sha256": sha256(args.summary),
|
||||
"corpus_manifest": str(args.corpus_manifest),
|
||||
"corpus_manifest_sha256": sha256(args.corpus_manifest),
|
||||
"fixture_mode": bool(args.fixture_mode),
|
||||
"proposal_model": str(args.proposal_model), "proposal_model_sha256": sha256(args.proposal_model),
|
||||
"device": args.device, "proposal_confidence": args.proposal_confidence, "crop_scale": args.crop_scale,
|
||||
"proposal_chunk_size": args.proposal_chunk_size,
|
||||
|
||||
@@ -53,6 +53,8 @@ if [[ -z "${PYTHON_BIN:-}" ]]; then
|
||||
fi
|
||||
fi
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
DATASET_YAML="${OPERATOR_YOLO_DATASET_DIR%/}/dataset.yaml"
|
||||
SUMMARY_PATH="${TRAIN_OUTPUT_DIR%/}/${TRAIN_RUN_NAME}/training_summary.json"
|
||||
export DATASET_YAML
|
||||
@@ -79,6 +81,11 @@ if [[ ! -f "${YOLO_BASE_MODEL_PATH}" ]]; then
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# A filename is not a training identity. Verify the immutable YAML/corpus/
|
||||
# asset/review release before importing Ultralytics or allocating CUDA work.
|
||||
"${PYTHON_BIN}" "${SCRIPT_DIR}/training_release_manifest.py" verify \
|
||||
--train-yaml "${DATASET_YAML}"
|
||||
|
||||
mkdir -p "${TRAIN_OUTPUT_DIR}" "$(dirname "${TRAIN_MODEL_OUTPUT_PATH}")"
|
||||
|
||||
"${PYTHON_BIN}" - <<'PY'
|
||||
|
||||
@@ -0,0 +1,545 @@
|
||||
"""Fail-closed provenance gate for Belgian building-training inputs.
|
||||
|
||||
The database data-contract validator decides whether a dataset can be stored.
|
||||
This module is intentionally a second, independent gate at the point where
|
||||
persisted datasets become irreversible training pairs. It has no database
|
||||
side effects, making the decision reproducible in a corpus manifest and easy
|
||||
to re-check before CUDA training starts.
|
||||
|
||||
Legacy data may only pass through this module in explicit fixture mode. That
|
||||
mode is deliberately limited to records marked as fixtures and must never be
|
||||
used for an operational corpus.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
from collections.abc import Mapping
|
||||
from pathlib import Path
|
||||
from typing import Any, Callable, Literal
|
||||
from uuid import UUID
|
||||
|
||||
|
||||
TRAINING_ELIGIBILITY_POLICY_VERSION = "geointel-training-source-eligibility/v1"
|
||||
DatasetRole = Literal["raster", "reference"]
|
||||
|
||||
_SHA256_RE = re.compile(r"^[0-9a-f]{64}$", re.IGNORECASE)
|
||||
_ALLOWED_SOURCE_CLASSIFICATIONS = {
|
||||
"authoritative",
|
||||
"corroborative",
|
||||
"contextual",
|
||||
"derived",
|
||||
}
|
||||
_FIXTURE_SOURCES = {"fixture", "test", "test_fixture", "test-fixture"}
|
||||
|
||||
|
||||
class TrainingEligibilityError(ValueError):
|
||||
"""Raised when persisted inputs cannot be used in an operational corpus."""
|
||||
|
||||
|
||||
class TrainingEligibilityResult:
|
||||
"""Stable, JSON-serializable decision for one persisted dataset."""
|
||||
|
||||
__slots__ = ("role", "eligible", "fixture_mode", "reasons", "evidence")
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
role: DatasetRole,
|
||||
eligible: bool,
|
||||
fixture_mode: bool,
|
||||
reasons: tuple[str, ...],
|
||||
evidence: dict[str, Any],
|
||||
) -> None:
|
||||
self.role = role
|
||||
self.eligible = eligible
|
||||
self.fixture_mode = fixture_mode
|
||||
self.reasons = reasons
|
||||
self.evidence = evidence
|
||||
|
||||
def as_dict(self) -> dict[str, Any]:
|
||||
return {
|
||||
"policy_version": TRAINING_ELIGIBILITY_POLICY_VERSION,
|
||||
"role": self.role,
|
||||
"eligible": self.eligible,
|
||||
"fixture_mode": self.fixture_mode,
|
||||
"reasons": list(self.reasons),
|
||||
"evidence": self.evidence,
|
||||
}
|
||||
|
||||
|
||||
def _value(record: Any, field: str, default: Any = None) -> Any:
|
||||
if isinstance(record, Mapping):
|
||||
return record.get(field, default)
|
||||
return getattr(record, field, default)
|
||||
|
||||
|
||||
def _normalise_status(value: Any) -> str:
|
||||
return str(value or "").strip().lower()
|
||||
|
||||
|
||||
def _normalise_mapping(value: Any) -> dict[str, Any]:
|
||||
return dict(value) if isinstance(value, Mapping) else {}
|
||||
|
||||
|
||||
def _source_identity(dataset: Any) -> tuple[str, str]:
|
||||
return (
|
||||
_normalise_status(_value(dataset, "source")),
|
||||
_normalise_status(_value(dataset, "source_name")),
|
||||
)
|
||||
|
||||
|
||||
def _is_explicit_fixture(dataset: Any, registry: Any) -> bool:
|
||||
source, source_name = _source_identity(dataset)
|
||||
metadata = _normalise_mapping(_value(dataset, "metadata_json"))
|
||||
provenance = _normalise_mapping(_value(dataset, "provenance_metadata"))
|
||||
usage_policy = _normalise_mapping(_value(registry, "usage_policy_json"))
|
||||
return bool(
|
||||
source in _FIXTURE_SOURCES
|
||||
or source_name in _FIXTURE_SOURCES
|
||||
or metadata.get("fixture") is True
|
||||
or provenance.get("fixture") is True
|
||||
or usage_policy.get("fixture_only") is True
|
||||
)
|
||||
|
||||
|
||||
def _dataset_evidence(dataset: Any, registry: Any, snapshot: Any) -> dict[str, Any]:
|
||||
usage_policy = _normalise_mapping(_value(registry, "usage_policy_json"))
|
||||
allowed_tasks = usage_policy.get("allowed_tasks")
|
||||
validation_authority = _normalise_mapping(usage_policy.get("validation_authority"))
|
||||
return {
|
||||
"dataset_id": str(_value(dataset, "id") or ""),
|
||||
"dataset_type": _normalise_status(_value(dataset, "dataset_type")),
|
||||
"dataset_role": _normalise_status(_value(dataset, "dataset_role")),
|
||||
"source": _normalise_status(_value(dataset, "source")),
|
||||
"source_name": _normalise_status(_value(dataset, "source_name")),
|
||||
"checksum_sha256": _value(dataset, "checksum_sha256"),
|
||||
"data_contract_key": _value(dataset, "data_contract_key"),
|
||||
"data_contract_version": _value(dataset, "data_contract_version"),
|
||||
"validation_status": _normalise_status(_value(dataset, "validation_status")),
|
||||
"provenance_status": _normalise_status(_value(dataset, "provenance_status")),
|
||||
"lineage_status": _normalise_status(_value(dataset, "lineage_status")),
|
||||
"quarantine_status": _normalise_status(_value(dataset, "quarantine_status")),
|
||||
"dataset_status": _normalise_status(_value(dataset, "status")),
|
||||
"dataset_source_registry_id": str(_value(dataset, "source_registry_id") or ""),
|
||||
"dataset_source_snapshot_id": str(_value(dataset, "source_snapshot_id") or ""),
|
||||
"source_registry_id": str(_value(registry, "id") or ""),
|
||||
"source_key": _normalise_status(_value(registry, "source_key")),
|
||||
"source_classification": _normalise_status(_value(registry, "classification")),
|
||||
"source_freshness_status": _normalise_status(_value(registry, "freshness_status")),
|
||||
"source_ingest_status": _normalise_status(_value(registry, "ingest_status")),
|
||||
"source_training_allowed": usage_policy.get("training_allowed"),
|
||||
"source_ground_truth_allowed": usage_policy.get("ground_truth_allowed"),
|
||||
"source_allowed_tasks": list(allowed_tasks) if isinstance(allowed_tasks, list) else [],
|
||||
"source_validation_authority": validation_authority,
|
||||
"source_building_validation_authority": validation_authority.get("building_validation"),
|
||||
"source_snapshot_id": str(_value(snapshot, "id") or _value(dataset, "source_snapshot_id") or ""),
|
||||
"snapshot_source_registry_id": str(_value(snapshot, "source_registry_id") or ""),
|
||||
"snapshot_key": _value(snapshot, "snapshot_key"),
|
||||
"snapshot_checksum_sha256": _value(snapshot, "checksum_sha256"),
|
||||
"snapshot_freshness_status": _normalise_status(_value(snapshot, "freshness_status")),
|
||||
"snapshot_ingest_status": _normalise_status(_value(snapshot, "ingest_status")),
|
||||
}
|
||||
|
||||
|
||||
def evaluate_dataset_training_eligibility(
|
||||
dataset: Any,
|
||||
*,
|
||||
role: DatasetRole,
|
||||
fixture_mode: bool = False,
|
||||
) -> TrainingEligibilityResult:
|
||||
"""Evaluate a source Dataset without mutating it.
|
||||
|
||||
Operational calls require server-attested source registry and snapshot
|
||||
relationships. ``fixture_mode`` can relax only legacy provenance fields,
|
||||
and only for an explicitly marked fixture. A failed validation or a
|
||||
quarantine is never relaxed.
|
||||
"""
|
||||
|
||||
registry = _value(dataset, "source_registry")
|
||||
snapshot = _value(dataset, "source_snapshot")
|
||||
evidence = _dataset_evidence(dataset, registry, snapshot)
|
||||
reasons: list[str] = []
|
||||
|
||||
dataset_type = evidence["dataset_type"]
|
||||
dataset_role = evidence["dataset_role"]
|
||||
if role == "raster" and dataset_type != "raster":
|
||||
reasons.append("dataset_type_not_raster")
|
||||
if role == "reference":
|
||||
if dataset_type != "vector":
|
||||
reasons.append("dataset_type_not_vector")
|
||||
if dataset_role != "reference":
|
||||
reasons.append("dataset_role_not_reference")
|
||||
|
||||
if evidence["dataset_status"] != "ready":
|
||||
reasons.append("dataset_not_ready")
|
||||
if evidence["quarantine_status"] == "quarantined":
|
||||
reasons.append("dataset_quarantined")
|
||||
|
||||
explicit_fixture = _is_explicit_fixture(dataset, registry)
|
||||
if fixture_mode and not explicit_fixture:
|
||||
reasons.append("fixture_mode_requires_explicit_fixture")
|
||||
validation_status = evidence["validation_status"]
|
||||
if validation_status == "failed":
|
||||
reasons.append("validation_failed")
|
||||
elif validation_status != "passed" and not (fixture_mode and explicit_fixture):
|
||||
reasons.append("validation_not_passed")
|
||||
|
||||
if not fixture_mode:
|
||||
if evidence["provenance_status"] != "complete":
|
||||
reasons.append("provenance_not_complete")
|
||||
if evidence["lineage_status"] not in {"complete", "not_applicable"}:
|
||||
reasons.append("lineage_not_complete")
|
||||
if not evidence["data_contract_key"] or not evidence["data_contract_version"]:
|
||||
reasons.append("data_contract_not_versioned")
|
||||
checksum = str(evidence["checksum_sha256"] or "")
|
||||
if not _SHA256_RE.fullmatch(checksum):
|
||||
reasons.append("dataset_checksum_invalid")
|
||||
if registry is None:
|
||||
reasons.append("source_registry_missing")
|
||||
if snapshot is None:
|
||||
reasons.append("source_snapshot_missing")
|
||||
if registry is not None:
|
||||
if (
|
||||
evidence["dataset_source_registry_id"]
|
||||
and evidence["dataset_source_registry_id"] != evidence["source_registry_id"]
|
||||
):
|
||||
reasons.append("dataset_source_registry_binding_mismatch")
|
||||
classification = evidence["source_classification"]
|
||||
if classification not in _ALLOWED_SOURCE_CLASSIFICATIONS:
|
||||
reasons.append("source_classification_not_allowed")
|
||||
if evidence["source_training_allowed"] is not True:
|
||||
reasons.append("source_not_allowed_for_training")
|
||||
if evidence["source_ingest_status"] == "quarantined":
|
||||
reasons.append("source_registry_quarantined")
|
||||
if snapshot is not None:
|
||||
if (
|
||||
evidence["dataset_source_snapshot_id"]
|
||||
and evidence["dataset_source_snapshot_id"] != evidence["source_snapshot_id"]
|
||||
):
|
||||
reasons.append("dataset_source_snapshot_binding_mismatch")
|
||||
if (
|
||||
registry is not None
|
||||
and evidence["snapshot_source_registry_id"]
|
||||
and evidence["snapshot_source_registry_id"] != evidence["source_registry_id"]
|
||||
):
|
||||
reasons.append("source_snapshot_registry_mismatch")
|
||||
if evidence["snapshot_ingest_status"] != "ingested":
|
||||
reasons.append("source_snapshot_not_ingested")
|
||||
if evidence["snapshot_freshness_status"] not in {"current", "not_applicable"}:
|
||||
reasons.append("source_snapshot_freshness_not_approved")
|
||||
snapshot_checksum = str(evidence["snapshot_checksum_sha256"] or "")
|
||||
if not _SHA256_RE.fullmatch(snapshot_checksum):
|
||||
reasons.append("source_snapshot_checksum_invalid")
|
||||
elif snapshot_checksum.lower() != checksum.lower():
|
||||
reasons.append("source_snapshot_checksum_mismatch")
|
||||
if role == "reference":
|
||||
if evidence["source_classification"] != "authoritative":
|
||||
reasons.append("reference_source_not_authoritative")
|
||||
if evidence["source_ground_truth_allowed"] is not True:
|
||||
reasons.append("reference_source_not_ground_truth_allowed")
|
||||
if "building_validation" not in evidence["source_allowed_tasks"]:
|
||||
reasons.append("reference_source_not_approved_for_building_validation")
|
||||
if evidence["source_building_validation_authority"] != "primary":
|
||||
reasons.append("reference_building_validation_not_primary")
|
||||
|
||||
return TrainingEligibilityResult(
|
||||
role=role,
|
||||
eligible=not reasons,
|
||||
fixture_mode=fixture_mode,
|
||||
reasons=tuple(sorted(set(reasons))),
|
||||
evidence=evidence,
|
||||
)
|
||||
|
||||
|
||||
def assert_dataset_training_eligible(
|
||||
dataset: Any,
|
||||
*,
|
||||
role: DatasetRole,
|
||||
fixture_mode: bool = False,
|
||||
sample_slug: str | None = None,
|
||||
) -> TrainingEligibilityResult:
|
||||
result = evaluate_dataset_training_eligibility(dataset, role=role, fixture_mode=fixture_mode)
|
||||
if result.eligible:
|
||||
return result
|
||||
label = f" for sample {sample_slug!r}" if sample_slug else ""
|
||||
raise TrainingEligibilityError(
|
||||
f"{role} dataset is not eligible for training{label}: {', '.join(result.reasons)}"
|
||||
)
|
||||
|
||||
|
||||
def training_pair_evidence(
|
||||
*,
|
||||
raster: Any,
|
||||
reference: Any,
|
||||
fixture_mode: bool = False,
|
||||
) -> dict[str, Any]:
|
||||
"""Return manifest-ready evidence for one raster/reference pair."""
|
||||
|
||||
raster_result = evaluate_dataset_training_eligibility(
|
||||
raster,
|
||||
role="raster",
|
||||
fixture_mode=fixture_mode,
|
||||
)
|
||||
reference_result = evaluate_dataset_training_eligibility(
|
||||
reference,
|
||||
role="reference",
|
||||
fixture_mode=fixture_mode,
|
||||
)
|
||||
return {
|
||||
"policy_version": TRAINING_ELIGIBILITY_POLICY_VERSION,
|
||||
"eligible": raster_result.eligible and reference_result.eligible,
|
||||
"fixture_mode": fixture_mode,
|
||||
"raster": raster_result.as_dict(),
|
||||
"reference": reference_result.as_dict(),
|
||||
}
|
||||
|
||||
|
||||
def manifest_training_eligibility_failures(
|
||||
manifest: Mapping[str, Any],
|
||||
*,
|
||||
fixture_mode: bool = False,
|
||||
) -> list[str]:
|
||||
"""Re-check immutable eligibility evidence before an actual training run."""
|
||||
|
||||
failures: list[str] = []
|
||||
eligibility = manifest.get("training_eligibility")
|
||||
if not isinstance(eligibility, Mapping):
|
||||
return ["manifest_training_eligibility_missing"]
|
||||
if eligibility.get("policy_version") != TRAINING_ELIGIBILITY_POLICY_VERSION:
|
||||
failures.append("manifest_training_eligibility_policy_invalid")
|
||||
if eligibility.get("status") != "eligible":
|
||||
failures.append("manifest_training_eligibility_not_eligible")
|
||||
manifest_fixture_mode = eligibility.get("fixture_mode") is True
|
||||
if manifest_fixture_mode != fixture_mode:
|
||||
failures.append("manifest_fixture_mode_mismatch")
|
||||
|
||||
samples = manifest.get("samples")
|
||||
if not isinstance(samples, list) or not samples:
|
||||
failures.append("manifest_samples_missing")
|
||||
return failures
|
||||
for sample in samples:
|
||||
if not isinstance(sample, Mapping):
|
||||
failures.append("manifest_sample_invalid")
|
||||
continue
|
||||
slug = str(sample.get("sample_slug") or "<unknown>")
|
||||
pair = sample.get("training_eligibility")
|
||||
if not isinstance(pair, Mapping):
|
||||
failures.append(f"{slug}:training_eligibility_missing")
|
||||
continue
|
||||
if pair.get("policy_version") != TRAINING_ELIGIBILITY_POLICY_VERSION:
|
||||
failures.append(f"{slug}:training_eligibility_policy_invalid")
|
||||
if pair.get("fixture_mode") is not fixture_mode:
|
||||
failures.append(f"{slug}:fixture_mode_mismatch")
|
||||
if pair.get("eligible") is not True:
|
||||
failures.append(f"{slug}:training_pair_not_eligible")
|
||||
for role in ("raster", "reference"):
|
||||
decision = pair.get(role)
|
||||
if not isinstance(decision, Mapping):
|
||||
failures.append(f"{slug}:{role}_eligibility_missing")
|
||||
continue
|
||||
if decision.get("eligible") is not True:
|
||||
failures.append(f"{slug}:{role}_not_eligible")
|
||||
reasons = decision.get("reasons")
|
||||
if isinstance(reasons, list) and reasons:
|
||||
failures.append(f"{slug}:{role}_has_rejection_reasons")
|
||||
return sorted(set(failures))
|
||||
|
||||
|
||||
def _default_live_dataset_access() -> tuple[Callable[[], Any], type[Any]]:
|
||||
"""Load the database dependencies only for an operational live re-check.
|
||||
|
||||
This module is also imported by pure filesystem tooling and fixture tests.
|
||||
Keeping the import lazy means those paths do not accidentally open a
|
||||
database connection, while a normal release/verify invocation still fails
|
||||
closed if the live registry cannot be checked.
|
||||
"""
|
||||
|
||||
import sys
|
||||
|
||||
repo_root = Path(__file__).resolve().parents[1]
|
||||
app_root = repo_root / "backend"
|
||||
if str(app_root) not in sys.path:
|
||||
sys.path.insert(0, str(app_root))
|
||||
from app.db.session import SessionLocal
|
||||
from app.models import Dataset
|
||||
|
||||
return SessionLocal, Dataset
|
||||
|
||||
|
||||
def live_manifest_training_eligibility_failures(
|
||||
manifest: Mapping[str, Any],
|
||||
*,
|
||||
fixture_mode: bool = False,
|
||||
session_factory: Callable[[], Any] | None = None,
|
||||
dataset_model: type[Any] | None = None,
|
||||
) -> list[str]:
|
||||
"""Re-evaluate every frozen corpus parent against the live database.
|
||||
|
||||
A frozen manifest proves what was eligible when it was created, not what
|
||||
remains eligible now. Operational releases therefore resolve the exact
|
||||
raster/reference Dataset ids again immediately before seal, retry, resume
|
||||
and CUDA training. A later quarantine, failed contract, snapshot change
|
||||
or missing record is a revocation and cannot be masked by the old manifest.
|
||||
|
||||
Fixture-only corpora deliberately have no operational authority and are
|
||||
never used for a production release; their isolated tests may skip this
|
||||
live database boundary.
|
||||
"""
|
||||
|
||||
if fixture_mode:
|
||||
return []
|
||||
failures: list[str] = []
|
||||
samples = manifest.get("samples")
|
||||
if not isinstance(samples, list) or not samples:
|
||||
return ["live_manifest_samples_missing"]
|
||||
if session_factory is None or dataset_model is None:
|
||||
try:
|
||||
default_factory, default_model = _default_live_dataset_access()
|
||||
except Exception:
|
||||
return ["live_training_dataset_access_unavailable"]
|
||||
session_factory = session_factory or default_factory
|
||||
dataset_model = dataset_model or default_model
|
||||
|
||||
try:
|
||||
db = session_factory()
|
||||
except Exception:
|
||||
return ["live_training_dataset_access_unavailable"]
|
||||
try:
|
||||
for sample in samples:
|
||||
if not isinstance(sample, Mapping):
|
||||
failures.append("live_manifest_sample_invalid")
|
||||
continue
|
||||
slug = str(sample.get("sample_slug") or "<unknown>")
|
||||
recorded_pair = sample.get("training_eligibility")
|
||||
for role, field_name in (("raster", "raster_dataset_id"), ("reference", "reference_dataset_id")):
|
||||
raw_id = sample.get(field_name)
|
||||
if not isinstance(raw_id, str) or not raw_id.strip():
|
||||
failures.append(f"{slug}:{role}_dataset_id_missing_for_live_check")
|
||||
continue
|
||||
try:
|
||||
dataset_id = UUID(raw_id)
|
||||
except (TypeError, ValueError, AttributeError):
|
||||
failures.append(f"{slug}:{role}_dataset_id_invalid_for_live_check")
|
||||
continue
|
||||
try:
|
||||
dataset = db.get(dataset_model, dataset_id)
|
||||
except Exception:
|
||||
failures.append(f"{slug}:{role}_live_lookup_failed")
|
||||
continue
|
||||
if dataset is None:
|
||||
failures.append(f"{slug}:{role}_live_dataset_missing")
|
||||
continue
|
||||
result = evaluate_dataset_training_eligibility(
|
||||
dataset,
|
||||
role=role, # type: ignore[arg-type]
|
||||
fixture_mode=False,
|
||||
)
|
||||
if not result.eligible:
|
||||
for reason in result.reasons:
|
||||
failures.append(f"{slug}:{role}_live_revoked:{reason}")
|
||||
if isinstance(recorded_pair, Mapping):
|
||||
recorded_role = recorded_pair.get(role)
|
||||
if isinstance(recorded_role, Mapping):
|
||||
recorded_evidence = recorded_role.get("evidence")
|
||||
if isinstance(recorded_evidence, Mapping) and str(recorded_evidence.get("dataset_id") or "") != raw_id:
|
||||
failures.append(f"{slug}:{role}_manifest_dataset_binding_mismatch")
|
||||
finally:
|
||||
close = getattr(db, "close", None)
|
||||
if callable(close):
|
||||
close()
|
||||
return sorted(set(failures))
|
||||
|
||||
|
||||
def assert_manifest_training_eligible(
|
||||
manifest: Mapping[str, Any],
|
||||
*,
|
||||
fixture_mode: bool = False,
|
||||
) -> None:
|
||||
failures = manifest_training_eligibility_failures(manifest, fixture_mode=fixture_mode)
|
||||
if failures:
|
||||
raise TrainingEligibilityError(
|
||||
"Corpus manifest is not eligible for training: " + ", ".join(failures)
|
||||
)
|
||||
|
||||
|
||||
def _file_sha256(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as stream:
|
||||
for chunk in iter(lambda: stream.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def frozen_manifest_training_eligibility_failures(
|
||||
manifest_path: Path,
|
||||
*,
|
||||
fixture_mode: bool = False,
|
||||
verify_live: bool = False,
|
||||
session_factory: Callable[[], Any] | None = None,
|
||||
dataset_model: type[Any] | None = None,
|
||||
) -> list[str]:
|
||||
"""Verify an immutable manifest and its source-eligibility decision together."""
|
||||
|
||||
failures: list[str] = []
|
||||
try:
|
||||
manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
|
||||
except (OSError, json.JSONDecodeError):
|
||||
return ["corpus_manifest_unreadable"]
|
||||
if not isinstance(manifest, Mapping):
|
||||
return ["corpus_manifest_invalid"]
|
||||
failures.extend(manifest_training_eligibility_failures(manifest, fixture_mode=fixture_mode))
|
||||
|
||||
freeze_path = manifest_path.parent / "corpus-freeze.json"
|
||||
try:
|
||||
freeze = json.loads(freeze_path.read_text(encoding="utf-8"))
|
||||
except (OSError, json.JSONDecodeError):
|
||||
return sorted(set(failures + ["corpus_freeze_missing_or_invalid"]))
|
||||
if not isinstance(freeze, Mapping):
|
||||
return sorted(set(failures + ["corpus_freeze_missing_or_invalid"]))
|
||||
if freeze.get("immutable") is not True:
|
||||
failures.append("corpus_freeze_not_immutable")
|
||||
if freeze.get("manifest_sha256") != _file_sha256(manifest_path):
|
||||
failures.append("corpus_manifest_checksum_mismatch")
|
||||
if freeze.get("training_eligibility_policy") != TRAINING_ELIGIBILITY_POLICY_VERSION:
|
||||
failures.append("corpus_freeze_policy_invalid")
|
||||
if bool(freeze.get("fixture_mode")) != fixture_mode:
|
||||
failures.append("corpus_freeze_fixture_mode_mismatch")
|
||||
if verify_live:
|
||||
failures.extend(
|
||||
live_manifest_training_eligibility_failures(
|
||||
manifest,
|
||||
fixture_mode=fixture_mode,
|
||||
session_factory=session_factory,
|
||||
dataset_model=dataset_model,
|
||||
)
|
||||
)
|
||||
return sorted(set(failures))
|
||||
|
||||
|
||||
def assert_frozen_manifest_training_eligible(
|
||||
manifest_path: Path,
|
||||
*,
|
||||
fixture_mode: bool = False,
|
||||
verify_live: bool = False,
|
||||
session_factory: Callable[[], Any] | None = None,
|
||||
dataset_model: type[Any] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Load only a frozen, policy-valid corpus manifest for a training entrypoint."""
|
||||
|
||||
failures = frozen_manifest_training_eligibility_failures(
|
||||
manifest_path,
|
||||
fixture_mode=fixture_mode,
|
||||
verify_live=verify_live,
|
||||
session_factory=session_factory,
|
||||
dataset_model=dataset_model,
|
||||
)
|
||||
if failures:
|
||||
raise TrainingEligibilityError(
|
||||
"Corpus manifest is not eligible for training: " + ", ".join(failures)
|
||||
)
|
||||
payload = json.loads(manifest_path.read_text(encoding="utf-8"))
|
||||
assert isinstance(payload, dict)
|
||||
return payload
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,738 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Exercise Phase-2 PostgreSQL migration guards on a disposable database.
|
||||
|
||||
This is intentionally *not* a general migration runner. It refuses every
|
||||
database whose name does not start with ``geointel_phase2_`` so it cannot be
|
||||
pointed accidentally at a developer, staging or production database. The
|
||||
script upgrades the complete Alembic chain, proves the new trigger guards with
|
||||
real DML, and optionally downgrades again.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
from contextlib import contextmanager
|
||||
from datetime import UTC, datetime
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
from typing import Any, Iterator
|
||||
from uuid import UUID, uuid4
|
||||
|
||||
from sqlalchemy import create_engine, text
|
||||
from sqlalchemy.engine import Connection, Engine, make_url
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
BACKEND = ROOT / "backend"
|
||||
SAFE_DATABASE_PREFIX = "geointel_phase2_"
|
||||
|
||||
|
||||
def _json_dump(path: Path, value: dict[str, Any]) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(
|
||||
json.dumps(value, indent=2, sort_keys=True) + "\n", encoding="utf-8"
|
||||
)
|
||||
|
||||
|
||||
def _require_disposable_database(database_url: str) -> None:
|
||||
parsed = make_url(database_url)
|
||||
database_name = str(parsed.database or "")
|
||||
if not database_name.startswith(SAFE_DATABASE_PREFIX):
|
||||
raise SystemExit(
|
||||
"Refusing migration guard test: database name must start with "
|
||||
f"{SAFE_DATABASE_PREFIX!r}, received {database_name!r}."
|
||||
)
|
||||
|
||||
|
||||
def _run_alembic(database_url: str, *arguments: str) -> str:
|
||||
environment = dict(os.environ)
|
||||
environment["DATABASE_URL"] = database_url
|
||||
completed = subprocess.run(
|
||||
[sys.executable, "-m", "alembic", *arguments],
|
||||
cwd=BACKEND,
|
||||
env=environment,
|
||||
text=True,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.STDOUT,
|
||||
check=False,
|
||||
)
|
||||
if completed.returncode != 0:
|
||||
raise RuntimeError(
|
||||
f"Alembic {' '.join(arguments)} failed with exit code {completed.returncode}:\n{completed.stdout}"
|
||||
)
|
||||
return completed.stdout
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _transaction(engine: Engine) -> Iterator[Connection]:
|
||||
with engine.begin() as connection:
|
||||
yield connection
|
||||
|
||||
|
||||
def _insert_project(connection: Connection) -> UUID:
|
||||
project_id = uuid4()
|
||||
connection.execute(
|
||||
text("INSERT INTO projects (id, name) VALUES (:id, :name)"),
|
||||
{"id": project_id, "name": "Phase 2 migration guard fixture"},
|
||||
)
|
||||
return project_id
|
||||
|
||||
|
||||
def _source_id(connection: Connection, source_key: str) -> UUID:
|
||||
value = connection.execute(
|
||||
text("SELECT id FROM source_registry WHERE source_key = :source_key"),
|
||||
{"source_key": source_key},
|
||||
).scalar_one()
|
||||
return UUID(str(value))
|
||||
|
||||
|
||||
def _insert_snapshot(
|
||||
connection: Connection, source_registry_id: UUID, *, key: str, checksum: str
|
||||
) -> UUID:
|
||||
snapshot_id = uuid4()
|
||||
connection.execute(
|
||||
text(
|
||||
"""
|
||||
INSERT INTO source_snapshots (
|
||||
id, source_registry_id, snapshot_key, checksum_sha256,
|
||||
crs, units, freshness_status, ingest_status
|
||||
) VALUES (
|
||||
:id, :source_registry_id, :snapshot_key, :checksum_sha256,
|
||||
'EPSG:4326', 'metres', 'current', 'ingested'
|
||||
)
|
||||
"""
|
||||
),
|
||||
{
|
||||
"id": snapshot_id,
|
||||
"source_registry_id": source_registry_id,
|
||||
"snapshot_key": key,
|
||||
"checksum_sha256": checksum,
|
||||
},
|
||||
)
|
||||
return snapshot_id
|
||||
|
||||
|
||||
def _accepted_report(
|
||||
*, contract_key: str = "geointel.vector.geojson", contract_version: str = "1.0.0"
|
||||
) -> str:
|
||||
"""Produce a structurally complete persisted validation report fixture."""
|
||||
|
||||
return json.dumps(
|
||||
{
|
||||
"asset_id": "phase2-guard-fixture",
|
||||
"data_contract_key": contract_key,
|
||||
"data_contract_version": contract_version,
|
||||
"contract_fingerprint_sha256": "d" * 64,
|
||||
"validation_status": "passed",
|
||||
"provenance_status": "complete",
|
||||
"lineage_status": "complete",
|
||||
"quarantine_status": "not_quarantined",
|
||||
"validation_scope": ["contract"],
|
||||
"report_sha256": "e" * 64,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _insert_dataset(
|
||||
connection: Connection,
|
||||
*,
|
||||
project_id: UUID,
|
||||
source_registry_id: UUID,
|
||||
source_snapshot_id: UUID,
|
||||
suffix: str,
|
||||
checksum_sha256: str,
|
||||
) -> UUID:
|
||||
dataset_id = uuid4()
|
||||
connection.execute(
|
||||
text(
|
||||
"""
|
||||
INSERT INTO datasets (
|
||||
id, project_id, name, dataset_type, source, source_name,
|
||||
status, source_registry_id, source_snapshot_id,
|
||||
data_contract_key, data_contract_version, validation_report_json,
|
||||
validation_status, provenance_status, lineage_status, quarantine_status,
|
||||
checksum_sha256
|
||||
) VALUES (
|
||||
:id, :project_id, :name, 'vector', 'governed', 'grb',
|
||||
'ready', :source_registry_id, :source_snapshot_id,
|
||||
'geointel.vector.geojson', '1.0.0', CAST(:validation_report_json AS json),
|
||||
'passed', 'complete', 'complete', 'not_quarantined', :checksum_sha256
|
||||
)
|
||||
"""
|
||||
),
|
||||
{
|
||||
"id": dataset_id,
|
||||
"project_id": project_id,
|
||||
"name": f"phase2-{suffix}.geojson",
|
||||
"source_registry_id": source_registry_id,
|
||||
"source_snapshot_id": source_snapshot_id,
|
||||
"validation_report_json": _accepted_report(),
|
||||
"checksum_sha256": checksum_sha256,
|
||||
},
|
||||
)
|
||||
return dataset_id
|
||||
|
||||
|
||||
def _assert_constraint_rejection(
|
||||
engine: Engine, statement: str, parameters: dict[str, Any], *, label: str
|
||||
) -> None:
|
||||
try:
|
||||
with _transaction(engine) as connection:
|
||||
connection.execute(text(statement), parameters)
|
||||
except IntegrityError as exc:
|
||||
sqlstate = getattr(getattr(exc, "orig", None), "sqlstate", None)
|
||||
if sqlstate == "23514":
|
||||
return
|
||||
raise AssertionError(
|
||||
f"{label} failed with unexpected SQLSTATE {sqlstate!r}"
|
||||
) from exc
|
||||
raise AssertionError(f"{label} was accepted unexpectedly")
|
||||
|
||||
|
||||
def verify(database_url: str, *, verify_downgrade: bool) -> dict[str, Any]:
|
||||
_require_disposable_database(database_url)
|
||||
started_at = datetime.now(UTC).isoformat()
|
||||
upgrade_output = _run_alembic(database_url, "upgrade", "head")
|
||||
engine = create_engine(database_url)
|
||||
try:
|
||||
edge_a = uuid4()
|
||||
edge_b = uuid4()
|
||||
with _transaction(engine) as connection:
|
||||
project_id = _insert_project(connection)
|
||||
grb_id = _source_id(connection, "grb")
|
||||
osm_id = _source_id(connection, "osm")
|
||||
grb_snapshot_id = _insert_snapshot(
|
||||
connection, grb_id, key="phase2-grb", checksum="a" * 64
|
||||
)
|
||||
osm_snapshot_id = _insert_snapshot(
|
||||
connection, osm_id, key="phase2-osm", checksum="b" * 64
|
||||
)
|
||||
derived_b_snapshot_id = _insert_snapshot(
|
||||
connection, grb_id, key="phase2-derived-b", checksum="c" * 64
|
||||
)
|
||||
derived_c_snapshot_id = _insert_snapshot(
|
||||
connection, grb_id, key="phase2-derived-c", checksum="d" * 64
|
||||
)
|
||||
mutation_snapshot_id = _insert_snapshot(
|
||||
connection, grb_id, key="phase2-mutation", checksum="e" * 64
|
||||
)
|
||||
dataset_a = _insert_dataset(
|
||||
connection,
|
||||
project_id=project_id,
|
||||
source_registry_id=grb_id,
|
||||
source_snapshot_id=grb_snapshot_id,
|
||||
suffix="a",
|
||||
checksum_sha256="a" * 64,
|
||||
)
|
||||
dataset_b = _insert_dataset(
|
||||
connection,
|
||||
project_id=project_id,
|
||||
source_registry_id=grb_id,
|
||||
source_snapshot_id=derived_b_snapshot_id,
|
||||
suffix="b",
|
||||
checksum_sha256="c" * 64,
|
||||
)
|
||||
dataset_c = _insert_dataset(
|
||||
connection,
|
||||
project_id=project_id,
|
||||
source_registry_id=grb_id,
|
||||
source_snapshot_id=derived_c_snapshot_id,
|
||||
suffix="c",
|
||||
checksum_sha256="d" * 64,
|
||||
)
|
||||
shared_dataset = _insert_dataset(
|
||||
connection,
|
||||
project_id=project_id,
|
||||
source_registry_id=grb_id,
|
||||
source_snapshot_id=grb_snapshot_id,
|
||||
suffix="shared",
|
||||
checksum_sha256="a" * 64,
|
||||
)
|
||||
mutation_dataset = _insert_dataset(
|
||||
connection,
|
||||
project_id=project_id,
|
||||
source_registry_id=grb_id,
|
||||
source_snapshot_id=mutation_snapshot_id,
|
||||
suffix="mutation",
|
||||
checksum_sha256="e" * 64,
|
||||
)
|
||||
version_id = uuid4()
|
||||
connection.execute(
|
||||
text(
|
||||
"""
|
||||
INSERT INTO dataset_versions (
|
||||
id, dataset_id, version, source_registry_id, source_snapshot_id,
|
||||
data_contract_key, data_contract_version, validation_report_json,
|
||||
validation_status, provenance_status, lineage_status, checksum_sha256
|
||||
) VALUES (
|
||||
:id, :dataset_id, 1, :source_registry_id, :source_snapshot_id,
|
||||
'geointel.vector.geojson', '1.0.0', CAST(:validation_report_json AS json),
|
||||
'passed', 'complete', 'complete', :checksum_sha256
|
||||
)
|
||||
"""
|
||||
),
|
||||
{
|
||||
"id": version_id,
|
||||
"dataset_id": dataset_a,
|
||||
"source_registry_id": grb_id,
|
||||
"source_snapshot_id": grb_snapshot_id,
|
||||
"validation_report_json": _accepted_report(),
|
||||
"checksum_sha256": "a" * 64,
|
||||
},
|
||||
)
|
||||
connection.execute(
|
||||
text(
|
||||
"""
|
||||
INSERT INTO dataset_lineage_edges (
|
||||
id, parent_dataset_id, child_dataset_id, relation_type, transformation_name
|
||||
) VALUES
|
||||
(:edge_a, :dataset_a, :dataset_b, 'derived_from', 'clip'),
|
||||
(:edge_b, :dataset_b, :dataset_c, 'derived_from', 'buffer')
|
||||
"""
|
||||
),
|
||||
{
|
||||
"edge_a": edge_a,
|
||||
"edge_b": edge_b,
|
||||
"dataset_a": dataset_a,
|
||||
"dataset_b": dataset_b,
|
||||
"dataset_c": dataset_c,
|
||||
},
|
||||
)
|
||||
|
||||
_assert_constraint_rejection(
|
||||
engine,
|
||||
"""
|
||||
INSERT INTO datasets (
|
||||
id, project_id, name, dataset_type, source, source_name, status,
|
||||
source_registry_id, source_snapshot_id, validation_status,
|
||||
data_contract_key, data_contract_version, validation_report_json,
|
||||
provenance_status, lineage_status, quarantine_status, checksum_sha256
|
||||
) VALUES (
|
||||
:id, :project_id, 'mismatched.geojson', 'vector', 'governed', 'grb', 'ready',
|
||||
:source_registry_id, :source_snapshot_id, 'passed',
|
||||
'geointel.vector.geojson', '1.0.0', CAST(:validation_report_json AS json),
|
||||
'complete', 'complete', 'not_quarantined', :checksum_sha256
|
||||
)
|
||||
""",
|
||||
{
|
||||
"id": uuid4(),
|
||||
"project_id": project_id,
|
||||
"source_registry_id": grb_id,
|
||||
"source_snapshot_id": osm_snapshot_id,
|
||||
"validation_report_json": _accepted_report(),
|
||||
"checksum_sha256": "b" * 64,
|
||||
},
|
||||
label="snapshot-registry mismatch guard",
|
||||
)
|
||||
_assert_constraint_rejection(
|
||||
engine,
|
||||
"""
|
||||
INSERT INTO datasets (
|
||||
id, project_id, name, dataset_type, source, source_name, status,
|
||||
source_registry_id, source_snapshot_id, validation_status,
|
||||
provenance_status, lineage_status, quarantine_status, checksum_sha256
|
||||
) VALUES (
|
||||
:id, :project_id, 'unreported.geojson', 'vector', 'governed', 'grb', 'ready',
|
||||
:source_registry_id, :source_snapshot_id, 'passed', 'complete', 'complete', 'not_quarantined', :checksum_sha256
|
||||
)
|
||||
""",
|
||||
{
|
||||
"id": uuid4(),
|
||||
"project_id": project_id,
|
||||
"source_registry_id": grb_id,
|
||||
"source_snapshot_id": grb_snapshot_id,
|
||||
"checksum_sha256": "a" * 64,
|
||||
},
|
||||
label="passed contract report guard",
|
||||
)
|
||||
_assert_constraint_rejection(
|
||||
engine,
|
||||
"""
|
||||
INSERT INTO datasets (
|
||||
id, project_id, name, dataset_type, source, source_name, status,
|
||||
source_registry_id, source_snapshot_id, validation_status,
|
||||
data_contract_key, data_contract_version, validation_report_json,
|
||||
provenance_status, lineage_status, quarantine_status
|
||||
) VALUES (
|
||||
:id, :project_id, 'missing-checksum.geojson', 'vector', 'governed', 'grb', 'ready',
|
||||
:source_registry_id, :source_snapshot_id, 'passed',
|
||||
'geointel.vector.geojson', '1.0.0', CAST(:validation_report_json AS json),
|
||||
'complete', 'complete', 'not_quarantined'
|
||||
)
|
||||
""",
|
||||
{
|
||||
"id": uuid4(),
|
||||
"project_id": project_id,
|
||||
"source_registry_id": grb_id,
|
||||
"source_snapshot_id": grb_snapshot_id,
|
||||
"validation_report_json": _accepted_report(),
|
||||
},
|
||||
label="passed dataset checksum required guard",
|
||||
)
|
||||
_assert_constraint_rejection(
|
||||
engine,
|
||||
"""
|
||||
INSERT INTO datasets (
|
||||
id, project_id, name, dataset_type, source, source_name, status,
|
||||
source_registry_id, source_snapshot_id, validation_status,
|
||||
data_contract_key, data_contract_version, validation_report_json,
|
||||
provenance_status, lineage_status, quarantine_status, checksum_sha256
|
||||
) VALUES (
|
||||
:id, :project_id, 'checksum-mismatch.geojson', 'vector', 'governed', 'grb', 'ready',
|
||||
:source_registry_id, :source_snapshot_id, 'passed',
|
||||
'geointel.vector.geojson', '1.0.0', CAST(:validation_report_json AS json),
|
||||
'complete', 'complete', 'not_quarantined', :checksum_sha256
|
||||
)
|
||||
""",
|
||||
{
|
||||
"id": uuid4(),
|
||||
"project_id": project_id,
|
||||
"source_registry_id": grb_id,
|
||||
"source_snapshot_id": grb_snapshot_id,
|
||||
"validation_report_json": _accepted_report(),
|
||||
"checksum_sha256": "f" * 64,
|
||||
},
|
||||
label="passed dataset snapshot checksum binding guard",
|
||||
)
|
||||
_assert_constraint_rejection(
|
||||
engine,
|
||||
"""
|
||||
INSERT INTO dataset_versions (
|
||||
id, dataset_id, version, source_registry_id, source_snapshot_id,
|
||||
data_contract_key, data_contract_version, validation_report_json,
|
||||
validation_status, provenance_status, lineage_status
|
||||
) VALUES (
|
||||
:id, :dataset_id, 2, :source_registry_id, :source_snapshot_id,
|
||||
'geointel.vector.geojson', '1.0.0', CAST(:validation_report_json AS json),
|
||||
'passed', 'complete', 'complete'
|
||||
)
|
||||
""",
|
||||
{
|
||||
"id": uuid4(),
|
||||
"dataset_id": dataset_a,
|
||||
"source_registry_id": grb_id,
|
||||
"source_snapshot_id": grb_snapshot_id,
|
||||
"validation_report_json": _accepted_report(),
|
||||
},
|
||||
label="passed dataset version checksum required guard",
|
||||
)
|
||||
_assert_constraint_rejection(
|
||||
engine,
|
||||
"""
|
||||
INSERT INTO dataset_versions (
|
||||
id, dataset_id, version, source_registry_id, source_snapshot_id,
|
||||
data_contract_key, data_contract_version, validation_report_json,
|
||||
validation_status, provenance_status, lineage_status, checksum_sha256
|
||||
) VALUES (
|
||||
:id, :dataset_id, 2, :source_registry_id, :source_snapshot_id,
|
||||
'geointel.vector.geojson', '1.0.0', CAST(:validation_report_json AS json),
|
||||
'passed', 'complete', 'complete', :checksum_sha256
|
||||
)
|
||||
""",
|
||||
{
|
||||
"id": uuid4(),
|
||||
"dataset_id": dataset_a,
|
||||
"source_registry_id": grb_id,
|
||||
"source_snapshot_id": grb_snapshot_id,
|
||||
"validation_report_json": _accepted_report(),
|
||||
"checksum_sha256": "f" * 64,
|
||||
},
|
||||
label="passed dataset version snapshot checksum binding guard",
|
||||
)
|
||||
_assert_constraint_rejection(
|
||||
engine,
|
||||
"UPDATE datasets SET data_contract_version = '9.9.9' WHERE id = :dataset_id",
|
||||
{"dataset_id": dataset_b},
|
||||
label="accepted contract evidence immutability guard",
|
||||
)
|
||||
_assert_constraint_rejection(
|
||||
engine,
|
||||
"UPDATE dataset_versions SET data_contract_version = '9.9.9' WHERE id = :dataset_version_id",
|
||||
{"dataset_version_id": version_id},
|
||||
label="accepted dataset-version contract evidence immutability guard",
|
||||
)
|
||||
_assert_constraint_rejection(
|
||||
engine,
|
||||
"UPDATE datasets SET storage_path = '/tampered/asset.geojson' WHERE id = :dataset_id",
|
||||
{"dataset_id": mutation_dataset},
|
||||
label="accepted dataset artifact immutability guard",
|
||||
)
|
||||
_assert_constraint_rejection(
|
||||
engine,
|
||||
"UPDATE datasets SET observed_at = CURRENT_TIMESTAMP WHERE id = :dataset_id",
|
||||
{"dataset_id": mutation_dataset},
|
||||
label="accepted dataset temporal evidence immutability guard",
|
||||
)
|
||||
_assert_constraint_rejection(
|
||||
engine,
|
||||
"UPDATE dataset_versions SET storage_path = '/tampered/version.geojson' WHERE id = :dataset_version_id",
|
||||
{"dataset_version_id": version_id},
|
||||
label="accepted dataset-version artifact immutability guard",
|
||||
)
|
||||
_assert_constraint_rejection(
|
||||
engine,
|
||||
"""
|
||||
UPDATE datasets
|
||||
SET validation_status = 'failed', checksum_sha256 = 'f' || repeat('0', 63)
|
||||
WHERE id = :dataset_id
|
||||
""",
|
||||
{"dataset_id": mutation_dataset},
|
||||
label="accepted dataset requires invalidation before artifact replacement",
|
||||
)
|
||||
with _transaction(engine) as connection:
|
||||
connection.execute(
|
||||
text(
|
||||
"UPDATE datasets SET validation_status = 'failed' WHERE id = :dataset_id"
|
||||
),
|
||||
{"dataset_id": mutation_dataset},
|
||||
)
|
||||
connection.execute(
|
||||
text(
|
||||
"UPDATE datasets SET storage_path = '/replacement/asset.geojson' WHERE id = :dataset_id"
|
||||
),
|
||||
{"dataset_id": mutation_dataset},
|
||||
)
|
||||
_assert_constraint_rejection(
|
||||
engine,
|
||||
"UPDATE source_snapshots SET source_registry_id = :registry_id WHERE id = :snapshot_id",
|
||||
{"registry_id": osm_id, "snapshot_id": grb_snapshot_id},
|
||||
label="immutable snapshot registry guard",
|
||||
)
|
||||
_assert_constraint_rejection(
|
||||
engine,
|
||||
"UPDATE source_registry SET display_name = 'tampered' WHERE id = :registry_id",
|
||||
{"registry_id": grb_id},
|
||||
label="server-owned source registry guard",
|
||||
)
|
||||
_assert_constraint_rejection(
|
||||
engine,
|
||||
"UPDATE source_snapshots SET checksum_sha256 = :checksum_sha256 WHERE id = :snapshot_id",
|
||||
{"checksum_sha256": "c" * 64, "snapshot_id": grb_snapshot_id},
|
||||
label="immutable snapshot evidence guard",
|
||||
)
|
||||
_assert_constraint_rejection(
|
||||
engine,
|
||||
"""
|
||||
INSERT INTO dataset_lineage_edges (
|
||||
id, parent_dataset_id, child_dataset_id, relation_type, transformation_name
|
||||
) VALUES (:id, :parent_dataset_id, :child_dataset_id, 'derived_from', 'cycle')
|
||||
""",
|
||||
{
|
||||
"id": uuid4(),
|
||||
"parent_dataset_id": dataset_c,
|
||||
"child_dataset_id": dataset_a,
|
||||
},
|
||||
label="lineage cycle guard",
|
||||
)
|
||||
_assert_constraint_rejection(
|
||||
engine,
|
||||
"UPDATE dataset_lineage_edges SET transformation_name = 'tampered' WHERE id = :id",
|
||||
{"id": edge_a},
|
||||
label="lineage edge immutability update guard",
|
||||
)
|
||||
_assert_constraint_rejection(
|
||||
engine,
|
||||
"DELETE FROM dataset_lineage_edges WHERE id = :id",
|
||||
{"id": edge_b},
|
||||
label="lineage edge immutability delete guard",
|
||||
)
|
||||
with _transaction(engine) as connection:
|
||||
connection.execute(
|
||||
text(
|
||||
"""
|
||||
INSERT INTO dataset_quarantines (
|
||||
id, dataset_version_id, stage, reason_code, status
|
||||
) VALUES (:id, :dataset_version_id, 'integration_test', 'CHECKSUM_MISMATCH', 'quarantined')
|
||||
"""
|
||||
),
|
||||
{"id": uuid4(), "dataset_version_id": version_id},
|
||||
)
|
||||
dataset_state = (
|
||||
connection.execute(
|
||||
text(
|
||||
"""
|
||||
SELECT status, quarantine_status, validation_status, provenance_status, lineage_status
|
||||
FROM datasets WHERE id = :dataset_id
|
||||
"""
|
||||
),
|
||||
{"dataset_id": dataset_a},
|
||||
)
|
||||
.mappings()
|
||||
.one()
|
||||
)
|
||||
version_state = (
|
||||
connection.execute(
|
||||
text(
|
||||
"""
|
||||
SELECT validation_status, provenance_status, lineage_status
|
||||
FROM dataset_versions WHERE id = :dataset_version_id
|
||||
"""
|
||||
),
|
||||
{"dataset_version_id": version_id},
|
||||
)
|
||||
.mappings()
|
||||
.one()
|
||||
)
|
||||
snapshot_state = connection.execute(
|
||||
text(
|
||||
"SELECT ingest_status FROM source_snapshots WHERE id = :snapshot_id"
|
||||
),
|
||||
{"snapshot_id": grb_snapshot_id},
|
||||
).scalar_one()
|
||||
child_dataset_state = (
|
||||
connection.execute(
|
||||
text(
|
||||
"""
|
||||
SELECT status, quarantine_status, validation_status, provenance_status, lineage_status
|
||||
FROM datasets WHERE id = :dataset_id
|
||||
"""
|
||||
),
|
||||
{"dataset_id": dataset_b},
|
||||
)
|
||||
.mappings()
|
||||
.one()
|
||||
)
|
||||
grandchild_dataset_state = (
|
||||
connection.execute(
|
||||
text(
|
||||
"""
|
||||
SELECT status, quarantine_status, validation_status, provenance_status, lineage_status
|
||||
FROM datasets WHERE id = :dataset_id
|
||||
"""
|
||||
),
|
||||
{"dataset_id": dataset_c},
|
||||
)
|
||||
.mappings()
|
||||
.one()
|
||||
)
|
||||
shared_dataset_state = (
|
||||
connection.execute(
|
||||
text(
|
||||
"""
|
||||
SELECT status, quarantine_status, validation_status, provenance_status, lineage_status
|
||||
FROM datasets WHERE id = :dataset_id
|
||||
"""
|
||||
),
|
||||
{"dataset_id": shared_dataset},
|
||||
)
|
||||
.mappings()
|
||||
.one()
|
||||
)
|
||||
expected_dataset_state = {
|
||||
"status": "quarantined",
|
||||
"quarantine_status": "quarantined",
|
||||
"validation_status": "failed",
|
||||
"provenance_status": "incomplete",
|
||||
"lineage_status": "incomplete",
|
||||
}
|
||||
if dict(dataset_state) != expected_dataset_state:
|
||||
raise AssertionError(
|
||||
f"quarantine parent propagation mismatch: {dict(dataset_state)}"
|
||||
)
|
||||
expected_version_state = {
|
||||
"validation_status": "failed",
|
||||
"provenance_status": "incomplete",
|
||||
"lineage_status": "incomplete",
|
||||
}
|
||||
if dict(version_state) != expected_version_state:
|
||||
raise AssertionError(
|
||||
f"quarantine version propagation mismatch: {dict(version_state)}"
|
||||
)
|
||||
if snapshot_state != "quarantined":
|
||||
raise AssertionError(
|
||||
f"quarantine snapshot propagation mismatch: {snapshot_state}"
|
||||
)
|
||||
if dict(child_dataset_state) != expected_dataset_state:
|
||||
raise AssertionError(
|
||||
"quarantine transitive-child propagation mismatch: "
|
||||
f"{dict(child_dataset_state)}"
|
||||
)
|
||||
if dict(grandchild_dataset_state) != expected_dataset_state:
|
||||
raise AssertionError(
|
||||
"quarantine transitive-grandchild propagation mismatch: "
|
||||
f"{dict(grandchild_dataset_state)}"
|
||||
)
|
||||
if dict(shared_dataset_state) != expected_dataset_state:
|
||||
raise AssertionError(
|
||||
"quarantine shared-snapshot propagation mismatch: "
|
||||
f"{dict(shared_dataset_state)}"
|
||||
)
|
||||
finally:
|
||||
engine.dispose()
|
||||
|
||||
downgrade_output = (
|
||||
_run_alembic(database_url, "downgrade", "base")
|
||||
if verify_downgrade
|
||||
else "not requested"
|
||||
)
|
||||
return {
|
||||
"schema_version": 1,
|
||||
"phase": "P2",
|
||||
"started_at": started_at,
|
||||
"completed_at": datetime.now(UTC).isoformat(),
|
||||
"migration_revision": "202608010001",
|
||||
"database_name": str(make_url(database_url).database),
|
||||
"result": "passed",
|
||||
"guards": {
|
||||
"snapshot_registry_pairing": "passed",
|
||||
"snapshot_registry_immutable": "passed",
|
||||
"source_registry_immutable": "passed",
|
||||
"snapshot_evidence_immutable": "passed",
|
||||
"lineage_cycle": "passed",
|
||||
"lineage_edge_immutable": "passed",
|
||||
"version_quarantine_propagation": "passed",
|
||||
"snapshot_quarantine_fanout": "passed",
|
||||
"transitive_lineage_quarantine": "passed",
|
||||
"passed_contract_report_required": "passed",
|
||||
"passed_dataset_checksum_required_and_snapshot_bound": "passed",
|
||||
"passed_dataset_version_checksum_required_and_snapshot_bound": "passed",
|
||||
"accepted_contract_evidence_immutable": "passed",
|
||||
"accepted_dataset_version_contract_evidence_immutable": "passed",
|
||||
"accepted_artifact_and_temporal_evidence_immutable": "passed",
|
||||
},
|
||||
"upgrade_output_tail": upgrade_output.splitlines()[-8:],
|
||||
"downgrade_output_tail": downgrade_output.splitlines()[-8:],
|
||||
}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--database-url", required=True)
|
||||
parser.add_argument(
|
||||
"--output",
|
||||
type=Path,
|
||||
default=ROOT
|
||||
/ "artifacts"
|
||||
/ "evidence"
|
||||
/ "accuracy"
|
||||
/ "P2"
|
||||
/ "postgres-migration-guards.json",
|
||||
)
|
||||
parser.add_argument("--skip-downgrade", action="store_true")
|
||||
args = parser.parse_args()
|
||||
try:
|
||||
result = verify(args.database_url, verify_downgrade=not args.skip_downgrade)
|
||||
except Exception as exc:
|
||||
result = {
|
||||
"schema_version": 1,
|
||||
"phase": "P2",
|
||||
"completed_at": datetime.now(UTC).isoformat(),
|
||||
"migration_revision": "202608010001",
|
||||
"result": "failed",
|
||||
"error_type": type(exc).__name__,
|
||||
"error": str(exc),
|
||||
}
|
||||
_json_dump(args.output, result)
|
||||
print(json.dumps(result, indent=2))
|
||||
return 1
|
||||
_json_dump(args.output, result)
|
||||
print(json.dumps(result, indent=2))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in New Issue
Block a user