audit: establish accuracy phase 1 baseline
This commit is contained in:
Jens
@@ -0,0 +1,364 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
from collections import Counter, defaultdict
|
||||
from datetime import datetime, timezone
|
||||
from itertools import combinations
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
def sha256_file(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as handle:
|
||||
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def artifact_record(path: Path, *, include_hash: bool = True) -> dict[str, Any]:
|
||||
stat = path.stat()
|
||||
return {
|
||||
"path": str(path),
|
||||
"size_bytes": stat.st_size,
|
||||
"modified_at": datetime.fromtimestamp(stat.st_mtime, timezone.utc).isoformat(),
|
||||
"sha256": sha256_file(path) if include_hash else None,
|
||||
}
|
||||
|
||||
|
||||
def manifest_summary(path: Path) -> dict[str, Any]:
|
||||
payload = json.loads(path.read_text(encoding="utf-8"))
|
||||
samples = payload.get("samples") if isinstance(payload.get("samples"), list) else []
|
||||
region_split = Counter(
|
||||
f"{sample.get('region')}/{sample.get('split')}" for sample in samples
|
||||
)
|
||||
contexts = Counter(str(sample.get("context")) for sample in samples)
|
||||
roles = Counter(str(sample.get("sample_role")) for sample in samples)
|
||||
background_samples = [
|
||||
{
|
||||
"sample_slug": sample.get("sample_slug"),
|
||||
"region": sample.get("region"),
|
||||
"reference_feature_count": sample.get("reference_feature_count"),
|
||||
"require_empty": sample.get("require_empty"),
|
||||
}
|
||||
for sample in samples
|
||||
if sample.get("split") == "background-test"
|
||||
]
|
||||
pure_empty = [
|
||||
sample
|
||||
for sample in background_samples
|
||||
if sample.get("require_empty") is True
|
||||
and int(sample.get("reference_feature_count") or 0) == 0
|
||||
]
|
||||
|
||||
cross_split_raster_hashes: dict[str, set[str]] = defaultdict(set)
|
||||
cross_split_dataset_ids: dict[str, set[str]] = defaultdict(set)
|
||||
for sample in samples:
|
||||
split = str(sample.get("split"))
|
||||
if sample.get("raster_sha256"):
|
||||
cross_split_raster_hashes[str(sample["raster_sha256"])].add(split)
|
||||
if sample.get("raster_dataset_id"):
|
||||
cross_split_dataset_ids[str(sample["raster_dataset_id"])].add(split)
|
||||
|
||||
spatial = cross_split_spatial_summary(samples)
|
||||
perceptual = cross_split_dhash_summary(samples)
|
||||
return {
|
||||
**artifact_record(path),
|
||||
"schema_version": payload.get("schema_version"),
|
||||
"dataset_version": payload.get("dataset_version"),
|
||||
"immutable": payload.get("immutable"),
|
||||
"sample_count": len(samples),
|
||||
"region_split_counts": dict(sorted(region_split.items())),
|
||||
"context_counts": dict(sorted(contexts.items())),
|
||||
"sample_role_counts": dict(sorted(roles.items())),
|
||||
"background_test_samples": background_samples,
|
||||
"pure_empty_background_count": len(pure_empty),
|
||||
"pure_empty_background_by_region": dict(
|
||||
sorted(Counter(str(item["region"]) for item in pure_empty).items())
|
||||
),
|
||||
"exact_cross_split_raster_hash_duplicate_count": sum(
|
||||
1 for splits in cross_split_raster_hashes.values() if len(splits) > 1
|
||||
),
|
||||
"cross_split_raster_dataset_id_duplicate_count": sum(
|
||||
1 for splits in cross_split_dataset_ids.values() if len(splits) > 1
|
||||
),
|
||||
"spatial_independence": spatial,
|
||||
"perceptual_duplicate_screen": perceptual,
|
||||
}
|
||||
|
||||
|
||||
def cross_split_spatial_summary(samples: list[dict[str, Any]]) -> dict[str, Any]:
|
||||
try:
|
||||
from pyproj import Transformer
|
||||
from shapely.geometry import box
|
||||
from shapely.ops import transform
|
||||
except ImportError:
|
||||
return {"available": False, "reason": "geospatial_dependencies_unavailable"}
|
||||
|
||||
transformer = Transformer.from_crs(4326, 31370, always_xy=True)
|
||||
records: list[tuple[str, str, Any]] = []
|
||||
for sample in samples:
|
||||
bbox_values = sample.get("bbox_epsg4326")
|
||||
if not isinstance(bbox_values, list) or len(bbox_values) != 4:
|
||||
continue
|
||||
geometry = transform(transformer.transform, box(*[float(value) for value in bbox_values]))
|
||||
records.append((str(sample.get("sample_slug")), str(sample.get("split")), geometry))
|
||||
|
||||
distances: list[tuple[float, str, str, str, str]] = []
|
||||
for left, right in combinations(records, 2):
|
||||
if left[1] == right[1]:
|
||||
continue
|
||||
distance = float(left[2].distance(right[2]))
|
||||
distances.append((distance, left[0], left[1], right[0], right[1]))
|
||||
distances.sort(key=lambda item: item[0])
|
||||
return {
|
||||
"available": True,
|
||||
"crs": "EPSG:31370",
|
||||
"cross_split_pair_count": len(distances),
|
||||
"minimum_distance_m": distances[0][0] if distances else None,
|
||||
"pairs_below_64_m": sum(1 for item in distances if item[0] < 64.0),
|
||||
"pairs_below_2000_m": sum(1 for item in distances if item[0] < 2000.0),
|
||||
"closest_pairs": [
|
||||
{
|
||||
"distance_m": item[0],
|
||||
"left_sample": item[1],
|
||||
"left_split": item[2],
|
||||
"right_sample": item[3],
|
||||
"right_split": item[4],
|
||||
}
|
||||
for item in distances[:20]
|
||||
],
|
||||
"claim_boundary": (
|
||||
"AOI bounding-box distance is a screening check only; it does not prove "
|
||||
"municipality, flight-strip, building-instance or imagery independence."
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def _dhash(path: Path) -> int:
|
||||
from PIL import Image
|
||||
|
||||
with Image.open(path) as image:
|
||||
resized = image.convert("L").resize((9, 8))
|
||||
pixel_source = (
|
||||
resized.get_flattened_data()
|
||||
if hasattr(resized, "get_flattened_data")
|
||||
else resized.getdata()
|
||||
)
|
||||
pixels = list(pixel_source)
|
||||
value = 0
|
||||
for row in range(8):
|
||||
offset = row * 9
|
||||
for column in range(8):
|
||||
value = (value << 1) | int(pixels[offset + column] > pixels[offset + column + 1])
|
||||
return value
|
||||
|
||||
|
||||
def cross_split_dhash_summary(samples: list[dict[str, Any]]) -> dict[str, Any]:
|
||||
hashes: list[tuple[str, str, int]] = []
|
||||
failures: list[dict[str, str]] = []
|
||||
for sample in samples:
|
||||
raster_path = Path(str(sample.get("raster_path") or ""))
|
||||
if not raster_path.is_file():
|
||||
failures.append({"sample_slug": str(sample.get("sample_slug")), "reason": "missing_raster"})
|
||||
continue
|
||||
try:
|
||||
hashes.append(
|
||||
(
|
||||
str(sample.get("sample_slug")),
|
||||
str(sample.get("split")),
|
||||
_dhash(raster_path),
|
||||
)
|
||||
)
|
||||
except Exception as exc:
|
||||
failures.append({"sample_slug": str(sample.get("sample_slug")), "reason": str(exc)})
|
||||
|
||||
matches: list[dict[str, Any]] = []
|
||||
minimum: int | None = None
|
||||
compared = 0
|
||||
for left, right in combinations(hashes, 2):
|
||||
if left[1] == right[1]:
|
||||
continue
|
||||
compared += 1
|
||||
distance = (left[2] ^ right[2]).bit_count()
|
||||
minimum = distance if minimum is None else min(minimum, distance)
|
||||
if distance <= 4:
|
||||
matches.append(
|
||||
{
|
||||
"left_sample": left[0],
|
||||
"left_split": left[1],
|
||||
"right_sample": right[0],
|
||||
"right_split": right[1],
|
||||
"hamming_distance": distance,
|
||||
}
|
||||
)
|
||||
return {
|
||||
"algorithm": "64-bit difference hash over 9x8 grayscale resize",
|
||||
"screened_raster_count": len(hashes),
|
||||
"cross_split_pair_count": compared,
|
||||
"minimum_hamming_distance": minimum,
|
||||
"pairs_at_or_below_4": matches,
|
||||
"failures": failures,
|
||||
"claim_boundary": (
|
||||
"This bounded perceptual screen is not semantic, instance-level or "
|
||||
"flight-strip deduplication and cannot establish split independence."
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def corpus_audit_summary(path: Path) -> dict[str, Any]:
|
||||
payload = json.loads(path.read_text(encoding="utf-8"))
|
||||
keys = (
|
||||
"status",
|
||||
"dataset_version",
|
||||
"manifest_immutable",
|
||||
"sample_count",
|
||||
"split_counts",
|
||||
"input_feature_count",
|
||||
"accepted_feature_count",
|
||||
"decision_counts",
|
||||
"temporal_unknown_sample_count",
|
||||
"spatial_leakage_status",
|
||||
"reviewed_sample_count",
|
||||
"review_complete",
|
||||
"failures",
|
||||
)
|
||||
return {
|
||||
**artifact_record(path),
|
||||
**{key: payload.get(key) for key in keys},
|
||||
"review_queue_count": len(payload.get("review_queue") or []),
|
||||
"claim_boundary": (
|
||||
"Automated status and an empty failure list do not substitute for "
|
||||
"human label acceptance."
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def calibration_summary(path: Path) -> dict[str, Any]:
|
||||
payload = json.loads(path.read_text(encoding="utf-8"))
|
||||
sweeps = payload.get("sweeps") if isinstance(payload.get("sweeps"), list) else []
|
||||
|
||||
def sweep_at(threshold: float) -> dict[str, Any] | None:
|
||||
return next(
|
||||
(
|
||||
{
|
||||
"threshold": item.get("threshold"),
|
||||
"aggregate": item.get("aggregate"),
|
||||
"regions": item.get("regions"),
|
||||
"pure_empty_false_positives": item.get("pure_empty_false_positives"),
|
||||
}
|
||||
for item in sweeps
|
||||
if abs(float(item.get("threshold")) - threshold) < 1e-12
|
||||
),
|
||||
None,
|
||||
)
|
||||
|
||||
return {
|
||||
**artifact_record(path),
|
||||
"model": payload.get("model"),
|
||||
"summary": payload.get("summary"),
|
||||
"split": payload.get("split"),
|
||||
"match_iou": payload.get("match_iou"),
|
||||
"tile_count": payload.get("tile_count"),
|
||||
"inference_imgsz": payload.get("inference_imgsz"),
|
||||
"sweep_count": len(sweeps),
|
||||
"threshold_0_15": sweep_at(0.15),
|
||||
"threshold_0_02": sweep_at(0.02),
|
||||
"claim_boundary": (
|
||||
"This is calibration-split, tile-level bbox evidence; it is not a "
|
||||
"protected-test, unique-building or national release result."
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def inventory(root: Path) -> dict[str, Any]:
|
||||
model_files = sorted(
|
||||
path
|
||||
for path in (root / "models").glob("*")
|
||||
if path.is_file() and path.suffix.lower() in {".pt", ".pth", ".onnx", ".engine", ".ckpt", ".safetensors"}
|
||||
)
|
||||
training_root = root / "storage" / "training"
|
||||
checkpoints = sorted(
|
||||
(
|
||||
path
|
||||
for path in training_root.glob("building-be-*/**/*")
|
||||
if path.is_file() and path.suffix.lower() in {".pt", ".pth", ".ckpt", ".onnx", ".engine", ".safetensors"}
|
||||
),
|
||||
key=lambda path: path.stat().st_mtime,
|
||||
reverse=True,
|
||||
)
|
||||
reports = sorted(
|
||||
(
|
||||
path
|
||||
for path in training_root.glob("building-be-*/**/*.json")
|
||||
if path.is_file()
|
||||
),
|
||||
key=lambda path: path.stat().st_mtime,
|
||||
reverse=True,
|
||||
)
|
||||
manifests = sorted(
|
||||
(root / "storage" / "operator-data").glob("building-be-*/operator_samples_manifest.json"),
|
||||
key=lambda path: path.stat().st_mtime,
|
||||
reverse=True,
|
||||
)
|
||||
return {
|
||||
"model_assets": [artifact_record(path) for path in model_files],
|
||||
"training_checkpoint_count": len(checkpoints),
|
||||
"training_checkpoint_bytes": sum(path.stat().st_size for path in checkpoints),
|
||||
"newest_training_checkpoints": [
|
||||
artifact_record(path, include_hash=False) for path in checkpoints[:30]
|
||||
],
|
||||
"training_json_report_count": len(reports),
|
||||
"newest_training_json_reports": [
|
||||
artifact_record(path, include_hash=False) for path in reports[:100]
|
||||
],
|
||||
"operator_manifest_count": len(manifests),
|
||||
"operator_manifests": [artifact_record(path) for path in manifests],
|
||||
}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Collect a bounded, read-only ML/data-lineage snapshot from the deployed GeoIntel volume."
|
||||
)
|
||||
parser.add_argument("--app-root", default="/app")
|
||||
args = parser.parse_args()
|
||||
root = Path(args.app_root).resolve()
|
||||
|
||||
paths = {
|
||||
"v56_audit": root / "storage/training/building-be-v56-corpus-audit-r1/belgium-building-corpus-audit.json",
|
||||
"v56_manifest": root / "storage/operator-data/building-be-v56-hard-negative-instance-roofs-r1/operator_samples_manifest.json",
|
||||
"v58_calibration": root / "storage/training/building-be-v58-v56-clean-pretrained-r1/preview-epoch-015/calibration.json",
|
||||
"v62_calibration": root / "storage/training/building-be-v62-v58-flanders-balanced-transfer-r1/preview-epoch-006/calibration-routed.json",
|
||||
"v66_manifest": root / "storage/operator-data/building-be-v66-lowrise-temporal-r1/operator_samples_manifest.json",
|
||||
}
|
||||
missing = [str(path) for path in paths.values() if not path.is_file()]
|
||||
if missing:
|
||||
raise SystemExit("Required lineage artifacts are missing: " + ", ".join(missing))
|
||||
|
||||
payload = {
|
||||
"schema_version": 1,
|
||||
"captured_at": datetime.now(timezone.utc).isoformat(),
|
||||
"read_only": True,
|
||||
"inventory": inventory(root),
|
||||
"v56": {
|
||||
"corpus_audit": corpus_audit_summary(paths["v56_audit"]),
|
||||
"manifest": manifest_summary(paths["v56_manifest"]),
|
||||
},
|
||||
"v58": {"calibration": calibration_summary(paths["v58_calibration"])},
|
||||
"v62": {"calibration": calibration_summary(paths["v62_calibration"])},
|
||||
"v66": {"manifest": manifest_summary(paths["v66_manifest"])},
|
||||
"global_claim_boundary": (
|
||||
"Inventory and historical calibration evidence do not prove human "
|
||||
"label acceptance, strict split independence, calibration, protected-test "
|
||||
"performance, national validity or release readiness."
|
||||
),
|
||||
}
|
||||
print(json.dumps(payload, indent=2, sort_keys=True))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in New Issue
Block a user