570 lines
30 KiB
Python
570 lines
30 KiB
Python
#!/usr/bin/env python3
|
|
"""Reproducible, read-only inventory and quarantine scan for Accuracy Phase 3.
|
|
|
|
The scanner deliberately operates on immutable source files and writes only to the
|
|
requested evidence directory. It is dependency-light, but uses rasterio and
|
|
shapely when available for type-specific checks. A checkpoint makes the scan
|
|
resumable without making source files mutable.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import hashlib
|
|
import json
|
|
import math
|
|
import os
|
|
import re
|
|
import tempfile
|
|
from collections import Counter, defaultdict
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
from typing import Any, Iterable
|
|
|
|
SCANNER_VERSION = "3.0.3"
|
|
SCHEMA_VERSION = 1
|
|
DEFAULT_ROOTS = ("models", "datasets", "data", "storage", "artifacts", "output")
|
|
EXCLUDED_DIRS = {".git", "node_modules", ".next", "__pycache__", ".pytest_cache"}
|
|
SEVERITIES = ("blocker", "critical", "major", "minor", "informational")
|
|
ACTIONS = (
|
|
"accept",
|
|
"repairable automatically",
|
|
"requires review",
|
|
"quarantine",
|
|
"exclude from training",
|
|
"exclude from evaluation",
|
|
"unavailable or unreadable",
|
|
)
|
|
|
|
|
|
def utc_now() -> str:
|
|
return datetime.now(timezone.utc).isoformat()
|
|
|
|
|
|
def sha256(path: Path) -> str:
|
|
digest = hashlib.sha256()
|
|
with path.open("rb") as stream:
|
|
for chunk in iter(lambda: stream.read(1024 * 1024), b""):
|
|
digest.update(chunk)
|
|
return digest.hexdigest()
|
|
|
|
|
|
def canonical_hash(value: Any) -> str:
|
|
return hashlib.sha256(
|
|
json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode("utf-8")
|
|
).hexdigest()
|
|
|
|
|
|
def atomic_json(path: Path, payload: Any) -> None:
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
fd, temp_name = tempfile.mkstemp(prefix=f".{path.name}.", suffix=".tmp", dir=path.parent)
|
|
try:
|
|
with os.fdopen(fd, "w", encoding="utf-8", newline="\n") as stream:
|
|
json.dump(payload, stream, ensure_ascii=False, indent=2, sort_keys=True)
|
|
stream.write("\n")
|
|
Path(temp_name).replace(path)
|
|
finally:
|
|
if os.path.exists(temp_name):
|
|
os.unlink(temp_name)
|
|
|
|
|
|
def severity_action(severity: str, action: str, code: str, message: str, *, field: str | None = None) -> dict[str, Any]:
|
|
if severity not in SEVERITIES or action not in ACTIONS:
|
|
raise ValueError(f"Unknown severity/action: {severity}/{action}")
|
|
result: dict[str, Any] = {"code": code, "severity": severity, "action": action, "message": message}
|
|
if field:
|
|
result["field"] = field
|
|
return result
|
|
|
|
|
|
def relative_path(path: Path, repo_root: Path) -> str:
|
|
return path.resolve(strict=False).relative_to(repo_root.resolve()).as_posix()
|
|
|
|
|
|
def is_excluded(path: Path, repo_root: Path, evidence_dir: Path) -> bool:
|
|
rel = relative_path(path, repo_root)
|
|
if any(part in EXCLUDED_DIRS for part in Path(rel).parts):
|
|
return True
|
|
evidence_rel = relative_path(evidence_dir, repo_root)
|
|
return rel == evidence_rel or rel.startswith(f"{evidence_rel}/")
|
|
|
|
|
|
def discover_files(repo_root: Path, roots: Iterable[str], evidence_dir: Path) -> list[Path]:
|
|
files: list[Path] = []
|
|
for root in roots:
|
|
base = (repo_root / root).resolve(strict=False)
|
|
if not base.exists():
|
|
continue
|
|
for path in base.rglob("*"):
|
|
if path.is_file() and not is_excluded(path, repo_root, evidence_dir):
|
|
files.append(path)
|
|
return sorted(set(files), key=lambda item: relative_path(item, repo_root))
|
|
|
|
|
|
def known_unreachable_items() -> list[dict[str, Any]]:
|
|
"""Boundaries identified by the Phase 1 inventory but not mounted locally."""
|
|
entries = (
|
|
("external://tower-corpora", "Tower corpora are not mounted in this project environment"),
|
|
("external://mounted-model-volumes", "Mounted model volumes are not available from this project environment"),
|
|
("external://production-postgis-or-api", "No production PostGIS/API endpoint is configured in the scan environment"),
|
|
)
|
|
result = []
|
|
for path, reason in entries:
|
|
result.append(
|
|
{
|
|
"item_id": hashlib.sha256(path.encode("utf-8")).hexdigest()[:20],
|
|
"path": path,
|
|
"status": "unreachable",
|
|
"read_status": "unavailable",
|
|
"size_bytes": None,
|
|
"modified_at": None,
|
|
"sha256": None,
|
|
"duplicate_group": None,
|
|
"near_duplicate_fingerprint": None,
|
|
"contract": {"key": "geointel.external.unavailable", "version": "1.0.0"},
|
|
"source": "unknown",
|
|
"lineage": None,
|
|
"schema_conformity": "unavailable",
|
|
"crs": None,
|
|
"units": None,
|
|
"resolution": None,
|
|
"spatial_coverage": None,
|
|
"temporal_coverage": None,
|
|
"freshness": {"status": "unavailable", "reason": reason},
|
|
"geometry_validation": None,
|
|
"empty_content": None,
|
|
"anomalies": [severity_action("critical", "unavailable or unreadable", "scope.unreachable", reason)],
|
|
"recommended_action": "unavailable or unreadable",
|
|
}
|
|
)
|
|
return result
|
|
|
|
|
|
def infer_source(rel: str, payload: Any = None) -> str:
|
|
text = rel.lower()
|
|
if isinstance(payload, dict):
|
|
text += " " + json.dumps(payload, ensure_ascii=False, sort_keys=True).lower()[:5000]
|
|
for token, name in (("grb", "GRB"), ("gebouwenregister", "Gebouwenregister"), ("dhmv", "DHMV"),
|
|
("sentinel", "Sentinel-2"), ("osm", "OSM"), ("orthofoto", "Orthophoto")):
|
|
if token in text:
|
|
return name
|
|
return "unknown"
|
|
|
|
|
|
def contract_for(path: Path, payload: Any = None) -> tuple[str, str]:
|
|
suffix = path.suffix.lower()
|
|
text = path.as_posix().lower()
|
|
if suffix in {".geojson", ".jsonl"} or "vector" in text:
|
|
return "geointel.vector.geojson", "1.0.0"
|
|
if suffix in {".tif", ".tiff", ".cog"}:
|
|
return "geointel.raster.geotiff", "1.0.0"
|
|
if suffix in {".pt", ".pth", ".onnx", ".safetensors"} or "model" in text and suffix not in {".json", ".md"}:
|
|
return "geointel.model.pytorch", "1.0.0"
|
|
if suffix == ".txt" and any(token in text for token in ("label", "yolo", "annotation")):
|
|
return "geointel.label.yolo", "1.1.0"
|
|
if suffix == ".db" or suffix in {".db-wal", ".db-shm"}:
|
|
return "geointel.database.sqlite", "1.0.0"
|
|
if suffix == ".json" and isinstance(payload, dict) and ("lineage" in payload or "schema_version" in payload):
|
|
return "geointel.manifest.json", "1.0.0"
|
|
return "geointel.artifact.generic", "1.0.0"
|
|
|
|
|
|
def add_anomaly(item: dict[str, Any], anomaly: dict[str, Any]) -> None:
|
|
item.setdefault("anomalies", []).append(anomaly)
|
|
|
|
|
|
def parse_json(path: Path) -> tuple[Any, str | None]:
|
|
try:
|
|
return json.loads(path.read_text(encoding="utf-8-sig")), None
|
|
except Exception as exc: # noqa: BLE001 - evidence must retain the concrete parser failure
|
|
return None, f"{type(exc).__name__}: {exc}"
|
|
|
|
|
|
def validate_geometry(item: dict[str, Any], payload: Any) -> None:
|
|
if not isinstance(payload, dict) or payload.get("type") != "FeatureCollection":
|
|
add_anomaly(item, severity_action("critical", "quarantine", "vector.invalid_schema", "Expected GeoJSON FeatureCollection"))
|
|
item["schema_conformity"] = "invalid"
|
|
return
|
|
features = payload.get("features")
|
|
if not isinstance(features, list):
|
|
add_anomaly(item, severity_action("critical", "quarantine", "vector.features_not_list", "GeoJSON features must be a list"))
|
|
item["schema_conformity"] = "invalid"
|
|
return
|
|
item["feature_count"] = len(features)
|
|
item["empty_content"] = len(features) == 0
|
|
if len(features) == 0:
|
|
add_anomaly(item, severity_action("minor", "requires review", "vector.empty", "GeoJSON contains no features"))
|
|
try:
|
|
from shapely.geometry import shape
|
|
except Exception:
|
|
item["geometry_validation"] = {"status": "unavailable", "reason": "shapely_not_installed"}
|
|
return
|
|
invalid = 0
|
|
empty = 0
|
|
bounds: list[float] | None = None
|
|
for index, feature in enumerate(features):
|
|
geometry = feature.get("geometry") if isinstance(feature, dict) else None
|
|
try:
|
|
geom = shape(geometry) if geometry else None
|
|
if geom is None or geom.is_empty:
|
|
empty += 1
|
|
elif not geom.is_valid:
|
|
invalid += 1
|
|
elif not geom.is_empty:
|
|
candidate = list(geom.bounds)
|
|
bounds = candidate if bounds is None else [min(bounds[0], candidate[0]), min(bounds[1], candidate[1]), max(bounds[2], candidate[2]), max(bounds[3], candidate[3])]
|
|
except Exception as exc: # noqa: BLE001
|
|
invalid += 1
|
|
if invalid == 1:
|
|
add_anomaly(item, severity_action("critical", "quarantine", "vector.geometry_parse_error", f"Feature {index}: {exc}"))
|
|
item["geometry_validation"] = {"status": "ok" if invalid == 0 else "invalid", "invalid_count": invalid, "empty_count": empty}
|
|
item["spatial_coverage"] = {"bbox": bounds, "crs": "EPSG:4326 (implicit GeoJSON)"}
|
|
if invalid:
|
|
add_anomaly(item, severity_action("critical", "quarantine", "vector.invalid_geometry", f"{invalid} invalid geometries"))
|
|
if bounds and (bounds[0] < -180 or bounds[2] > 180 or bounds[1] < -90 or bounds[3] > 90):
|
|
add_anomaly(item, severity_action("critical", "quarantine", "vector.bbox_out_of_range", "EPSG:4326 coordinates exceed valid bounds", field="bbox"))
|
|
|
|
|
|
def validate_raster(item: dict[str, Any], path: Path) -> None:
|
|
try:
|
|
import rasterio
|
|
import numpy as np
|
|
with rasterio.open(path) as dataset:
|
|
item["raster"] = {
|
|
"width": dataset.width, "height": dataset.height, "bands": dataset.count,
|
|
"dtype": list(dataset.dtypes), "crs": dataset.crs.to_string() if dataset.crs else None,
|
|
"resolution": [float(dataset.res[0]), float(dataset.res[1])],
|
|
"bounds": [float(v) for v in dataset.bounds], "nodata": dataset.nodata,
|
|
}
|
|
item["crs"] = item["raster"]["crs"]
|
|
item["resolution"] = item["raster"]["resolution"]
|
|
item["spatial_coverage"] = {"bbox": item["raster"]["bounds"], "crs": item["crs"]}
|
|
item["empty_content"] = dataset.width == 0 or dataset.height == 0
|
|
if not dataset.crs:
|
|
add_anomaly(item, severity_action("critical", "quarantine", "raster.missing_crs", "Raster has no CRS"))
|
|
if dataset.width <= 0 or dataset.height <= 0:
|
|
add_anomaly(item, severity_action("critical", "quarantine", "raster.empty_dimensions", "Raster has invalid dimensions"))
|
|
sample = dataset.read(1, masked=True, out_shape=(1, min(dataset.height, 256), min(dataset.width, 256)))
|
|
item["raster"]["sample_valid_fraction"] = float(np.ma.count(sample) / max(sample.size, 1))
|
|
if item["raster"]["sample_valid_fraction"] == 0:
|
|
add_anomaly(item, severity_action("major", "quarantine", "raster.all_nodata", "Raster sample contains no valid pixels"))
|
|
if any(not math.isfinite(value) for value in item["raster"]["resolution"]) or min(item["raster"]["resolution"]) <= 0:
|
|
add_anomaly(item, severity_action("critical", "quarantine", "raster.invalid_resolution", "Raster resolution is non-positive or non-finite"))
|
|
except Exception as exc: # noqa: BLE001
|
|
item["read_status"] = "unreadable"
|
|
add_anomaly(item, severity_action("critical", "unavailable or unreadable", "raster.unreadable", f"{type(exc).__name__}: {exc}"))
|
|
|
|
|
|
def validate_manifest(item: dict[str, Any], payload: Any, repo_root: Path) -> None:
|
|
if not isinstance(payload, dict):
|
|
return
|
|
if "crs" in payload:
|
|
item["crs"] = payload.get("crs")
|
|
if "bounds" in payload:
|
|
item["spatial_coverage"] = {"bbox": payload.get("bounds"), "crs": payload.get("crs")}
|
|
bounds = payload.get("bounds")
|
|
if isinstance(bounds, list) and len(bounds) == 4 and all(float(v) == 0 for v in bounds):
|
|
add_anomaly(item, severity_action("major", "quarantine", "manifest.zero_bbox", "Manifest declares a zero-area bounding box", field="bounds"))
|
|
if "tile_paths" in payload and isinstance(payload["tile_paths"], list):
|
|
missing = []
|
|
for raw in payload["tile_paths"]:
|
|
candidate = Path(str(raw))
|
|
if not candidate.is_absolute():
|
|
candidate = repo_root / candidate
|
|
if not candidate.exists():
|
|
missing.append(str(raw))
|
|
item["lineage"] = {"source_dataset_id": payload.get("source_dataset_id"), "source_raster_id": payload.get("source_raster_id")}
|
|
if missing:
|
|
add_anomaly(item, severity_action("critical", "quarantine", "manifest.missing_tile", f"{len(missing)} tile paths are absent"))
|
|
item["missing_tile_paths"] = missing
|
|
|
|
|
|
def validate_label(item: dict[str, Any], path: Path) -> None:
|
|
try:
|
|
lines = [line.strip() for line in path.read_text(encoding="utf-8").splitlines() if line.strip()]
|
|
except Exception as exc: # noqa: BLE001
|
|
add_anomaly(item, severity_action("critical", "unavailable or unreadable", "label.unreadable", str(exc)))
|
|
return
|
|
invalid = 0
|
|
for line in lines:
|
|
parts = line.split()
|
|
try:
|
|
if len(parts) != 5 or int(parts[0]) < 0 or any(not 0 <= float(value) <= 1 for value in parts[1:]):
|
|
invalid += 1
|
|
except ValueError:
|
|
invalid += 1
|
|
item["label"] = {"line_count": len(lines), "invalid_line_count": invalid}
|
|
if invalid:
|
|
add_anomaly(item, severity_action("critical", "quarantine", "label.invalid_yolo", f"{invalid} label lines violate YOLO normalized schema"))
|
|
if not lines:
|
|
item["empty_content"] = True
|
|
add_anomaly(item, severity_action("minor", "requires review", "label.empty", "Empty label file requires explicit background review evidence"))
|
|
|
|
|
|
def json_metadata(item: dict[str, Any], payload: Any) -> None:
|
|
if not isinstance(payload, dict):
|
|
return
|
|
observed = next((payload.get(key) for key in ("observed_at", "created_at", "updated_at", "timestamp", "fetched_at") if payload.get(key)), None)
|
|
if observed:
|
|
item["freshness"] = {"observed_at": observed, "method": "declared_metadata", "status": "declared"}
|
|
lineage_keys = [key for key in payload if "lineage" in key.lower() or key.startswith("source_") or key in {"parent_id", "parent_dataset_id"}]
|
|
if lineage_keys:
|
|
item["lineage"] = {key: payload.get(key) for key in lineage_keys}
|
|
|
|
|
|
def normalized_fingerprint(path: Path, payload: Any) -> str | None:
|
|
if payload is not None:
|
|
return canonical_hash(payload)
|
|
if path.suffix.lower() not in {".png", ".jpg", ".jpeg", ".webp"}:
|
|
return None
|
|
try:
|
|
from PIL import Image
|
|
image = Image.open(path).convert("L").resize((16, 16))
|
|
pixels = list(image.getdata())
|
|
average = sum(pixels) / max(len(pixels), 1)
|
|
return "image:" + "".join("1" if pixel >= average else "0" for pixel in pixels)
|
|
except Exception:
|
|
return None
|
|
|
|
|
|
def make_item(path: Path, repo_root: Path) -> dict[str, Any]:
|
|
rel = relative_path(path, repo_root)
|
|
stat = path.stat()
|
|
item: dict[str, Any] = {
|
|
"item_id": hashlib.sha256(rel.encode("utf-8")).hexdigest()[:20],
|
|
"path": rel,
|
|
"status": "examined",
|
|
"read_status": "readable",
|
|
"size_bytes": stat.st_size,
|
|
"modified_at": datetime.fromtimestamp(stat.st_mtime, tz=timezone.utc).isoformat(),
|
|
"sha256": None,
|
|
"duplicate_group": None,
|
|
"near_duplicate_fingerprint": None,
|
|
"contract": None,
|
|
"source": infer_source(rel),
|
|
"lineage": None,
|
|
"schema_conformity": "not_applicable",
|
|
"crs": None,
|
|
"units": None,
|
|
"resolution": None,
|
|
"spatial_coverage": None,
|
|
"temporal_coverage": None,
|
|
"freshness": {"status": "unknown", "reason": "no_declared_source_timestamp"},
|
|
"geometry_validation": None,
|
|
"empty_content": stat.st_size == 0,
|
|
"anomalies": [],
|
|
}
|
|
try:
|
|
item["sha256"] = sha256(path)
|
|
except Exception as exc: # noqa: BLE001
|
|
item["status"] = "unreachable"
|
|
item["read_status"] = "unreadable"
|
|
add_anomaly(item, severity_action("critical", "unavailable or unreadable", "file.unreadable", f"{type(exc).__name__}: {exc}"))
|
|
return item
|
|
payload = None
|
|
parse_error = None
|
|
if path.suffix.lower() in {".json", ".geojson"} or path.name.lower().endswith(".jsonl"):
|
|
payload, parse_error = parse_json(path)
|
|
if parse_error:
|
|
item["schema_conformity"] = "invalid"
|
|
add_anomaly(item, severity_action("critical", "quarantine", "json.invalid", parse_error))
|
|
else:
|
|
json_metadata(item, payload)
|
|
contract_key, contract_version = contract_for(path, payload)
|
|
item["contract"] = {"key": contract_key, "version": contract_version}
|
|
item["source"] = infer_source(rel, payload)
|
|
if path.suffix.lower() == ".geojson" and payload is not None:
|
|
item["schema_conformity"] = "valid"
|
|
validate_geometry(item, payload)
|
|
elif path.suffix.lower() in {".tif", ".tiff", ".cog"}:
|
|
validate_raster(item, path)
|
|
item["schema_conformity"] = "valid" if item["read_status"] == "readable" else "invalid"
|
|
elif path.name.lower() == "manifest.json" and payload is not None:
|
|
item["schema_conformity"] = "valid" if isinstance(payload, dict) else "invalid"
|
|
validate_manifest(item, payload, repo_root)
|
|
elif contract_key == "geointel.label.yolo" and path.suffix.lower() == ".txt":
|
|
item["schema_conformity"] = "valid"
|
|
validate_label(item, path)
|
|
if item["source"] == "OSM":
|
|
item["ground_truth_eligible"] = False
|
|
add_anomaly(item, severity_action("informational", "requires review", "source.osm_not_ground_truth", "OSM is corroborative/contextual and never automatic ground truth"))
|
|
if item["lineage"] is None and ("derived" in rel.lower() or "result" in rel.lower() or "calibration" in rel.lower()):
|
|
add_anomaly(item, severity_action("major", "quarantine", "lineage.missing", "Derived-looking artifact has no explicit lineage"))
|
|
if item["freshness"].get("status") == "unknown" and item["source"] not in {"unknown", "Orthophoto"}:
|
|
add_anomaly(item, severity_action("minor", "requires review", "freshness.missing", "Source-labelled item has no declared observation timestamp"))
|
|
item["near_duplicate_fingerprint"] = normalized_fingerprint(path, payload)
|
|
if item["anomalies"]:
|
|
item["recommended_action"] = sorted(item["anomalies"], key=lambda a: SEVERITIES.index(a["severity"]))[0]["action"]
|
|
else:
|
|
item["recommended_action"] = "accept"
|
|
return item
|
|
|
|
|
|
def detect_duplicates(items: list[dict[str, Any]]) -> dict[str, Any]:
|
|
by_hash: dict[str, list[str]] = defaultdict(list)
|
|
by_fingerprint: dict[str, list[str]] = defaultdict(list)
|
|
for item in items:
|
|
if item.get("sha256"):
|
|
by_hash[item["sha256"]].append(item["path"])
|
|
if item.get("near_duplicate_fingerprint"):
|
|
by_fingerprint[item["near_duplicate_fingerprint"]].append(item["path"])
|
|
exact = {key: sorted(paths) for key, paths in by_hash.items() if len(paths) > 1}
|
|
near = {key: sorted(paths) for key, paths in by_fingerprint.items() if len(paths) > 1}
|
|
for group, paths in exact.items():
|
|
for item in items:
|
|
if item["path"] in paths:
|
|
item["duplicate_group"] = group
|
|
return {"schema_version": 1, "exact_duplicate_groups": exact, "near_duplicate_groups": near, "method": "sha256_and_normalized_payload_or_image_ahash"}
|
|
|
|
|
|
def detect_leakage(items: list[dict[str, Any]], repo_root: Path | None = None) -> dict[str, Any]:
|
|
split_by_hash: dict[str, set[str]] = defaultdict(set)
|
|
split_by_bbox: dict[str, list[tuple[str, list[float], str]]] = defaultdict(list)
|
|
for item in items:
|
|
text = item["path"].lower()
|
|
split = next((candidate for candidate in ("train", "val", "validation", "calibration", "test", "background-test") if re.search(rf"(?:^|[/_.-]){re.escape(candidate)}(?:[/_.-]|$)", text)), None)
|
|
if not split:
|
|
continue
|
|
if item.get("sha256"):
|
|
split_by_hash[item["sha256"]].add(split)
|
|
bbox = (item.get("spatial_coverage") or {}).get("bbox") if isinstance(item.get("spatial_coverage"), dict) else None
|
|
if isinstance(bbox, list) and len(bbox) == 4:
|
|
try:
|
|
split_by_bbox[split].append((item["path"], [float(value) for value in bbox], split))
|
|
except (TypeError, ValueError):
|
|
pass
|
|
cross_hash = {key: sorted(value) for key, value in split_by_hash.items() if len(value) > 1}
|
|
source_audits = [item["path"] for item in items if item["path"].endswith("spatial-leakage-audit.json")]
|
|
prior_quality: dict[str, Any] | None = None
|
|
prior_attention = False
|
|
if repo_root is not None:
|
|
status_path = repo_root / "docs/accuracy-program/status.json"
|
|
try:
|
|
status = json.loads(status_path.read_text(encoding="utf-8"))
|
|
v56 = ((status.get("ml_data") or {}).get("v56") or {})
|
|
prior_quality = {
|
|
"path": relative_path(status_path, repo_root),
|
|
"cross_split_pairs_below_2000_m": v56.get("cross_split_pairs_below_2000_m"),
|
|
"split_independence_proven": v56.get("split_independence_proven"),
|
|
"exact_cross_split_raster_hash_duplicates": v56.get("exact_cross_split_raster_hash_duplicates"),
|
|
}
|
|
prior_attention = (prior_quality.get("cross_split_pairs_below_2000_m") or 0) > 0 or prior_quality.get("split_independence_proven") is False
|
|
except (OSError, json.JSONDecodeError):
|
|
prior_quality = None
|
|
return {
|
|
"schema_version": 1,
|
|
"status": "attention" if cross_hash or source_audits or prior_attention else "no_detected_overlap",
|
|
"same_checksum_across_splits": cross_hash,
|
|
"spatial_overlap_checks": "not_proven_without_AOI_split_geometry",
|
|
"source_spatial_audit_files": source_audits,
|
|
"prior_quality_inventory": prior_quality,
|
|
"limitations": ["Filename-derived split tokens are conservative; AOI independence requires authoritative split geometry."],
|
|
}
|
|
|
|
|
|
def report_payload(items: list[dict[str, Any]], inventory: dict[str, Any], scan_id: str, *, started_at: str, completed_at: str, repo_root: Path | None = None) -> dict[str, Any]:
|
|
anomalies = [dict({"path": item["path"], "item_id": item["item_id"]}, **anomaly) for item in items for anomaly in item.get("anomalies", [])]
|
|
quarantine = [
|
|
{"path": item["path"], "item_id": item["item_id"], "recommended_action": item.get("recommended_action"), "anomalies": item.get("anomalies", [])}
|
|
for item in items if item.get("recommended_action") in {"quarantine", "exclude from training", "exclude from evaluation", "unavailable or unreadable"}
|
|
]
|
|
duplicates = detect_duplicates(items)
|
|
leakage = detect_leakage(items, repo_root)
|
|
category_counts = Counter(anomaly["code"] for anomaly in anomalies)
|
|
dataset_summary: dict[str, dict[str, Any]] = {}
|
|
for item in items:
|
|
key = item["contract"]["key"] if item.get("contract") else "unknown"
|
|
block = dataset_summary.setdefault(key, {"item_count": 0, "anomaly_count": 0, "paths": [], "severity_counts": Counter()})
|
|
block["item_count"] += 1
|
|
block["paths"].append(item["path"])
|
|
block["anomaly_count"] += len(item.get("anomalies", []))
|
|
block["severity_counts"].update(anomaly["severity"] for anomaly in item.get("anomalies", []))
|
|
for block in dataset_summary.values():
|
|
block["paths"] = sorted(block["paths"])
|
|
block["severity_counts"] = dict(sorted(block["severity_counts"].items()))
|
|
source_freshness = Counter((item.get("source", "unknown"), (item.get("freshness") or {}).get("status", "unknown")) for item in items)
|
|
counts = Counter(item.get("status", "examined") for item in items)
|
|
reconciliation = {"examined": counts.get("examined", 0), "skipped": counts.get("skipped", 0), "unreachable": counts.get("unreachable", 0), "inventory_total": len(items), "reconciles": sum(counts.get(key, 0) for key in ("examined", "skipped", "unreachable")) == len(items)}
|
|
return {
|
|
"schema_version": SCHEMA_VERSION, "scanner_version": SCANNER_VERSION, "scan_id": scan_id,
|
|
"started_at": started_at, "completed_at": completed_at, "inventory": inventory,
|
|
"reconciliation": reconciliation, "items": sorted(items, key=lambda item: item["path"]),
|
|
"anomaly_count": len(anomalies), "anomalies": sorted(anomalies, key=lambda item: (item["path"], item["code"])),
|
|
"quarantine": sorted(quarantine, key=lambda item: item["path"]),
|
|
"duplicates": duplicates, "leakage": leakage,
|
|
"dataset_summary": {key: dataset_summary[key] for key in sorted(dataset_summary)},
|
|
"source_freshness": {f"{source}|{status}": count for (source, status), count in sorted(source_freshness.items())},
|
|
"anomaly_category_counts": dict(sorted(category_counts.items())),
|
|
"grb_consistency": {"status": "unavailable", "reason": "No authoritative GRB snapshot was accessible in the configured local roots; no derived result was marked as GRB ground truth."},
|
|
"determinism": {"content_hash": canonical_hash({"inventory": inventory, "items": sorted(items, key=lambda item: item["path"]), "anomalies": sorted(anomalies, key=lambda item: (item["path"], item["code"]))}), "timestamps_excluded_from_content_hash": True},
|
|
}
|
|
|
|
|
|
def parse_args() -> argparse.Namespace:
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--repo-root", type=Path, default=Path(__file__).resolve().parents[1])
|
|
parser.add_argument("--output-dir", type=Path, default=None)
|
|
parser.add_argument("--batch-size", type=int, default=50)
|
|
parser.add_argument("--resume", action="store_true")
|
|
parser.add_argument("--roots", nargs="+", default=list(DEFAULT_ROOTS))
|
|
return parser.parse_args()
|
|
|
|
|
|
def main() -> int:
|
|
args = parse_args()
|
|
repo_root = args.repo_root.resolve()
|
|
output_dir = (args.output_dir or repo_root / "artifacts/evidence/accuracy/P3").resolve()
|
|
if args.batch_size <= 0:
|
|
raise SystemExit("--batch-size must be positive")
|
|
paths = discover_files(repo_root, args.roots, output_dir)
|
|
inventory_entries = [{"path": relative_path(path, repo_root), "size_bytes": path.stat().st_size, "modified_ns": path.stat().st_mtime_ns} for path in paths]
|
|
unreachable_items = known_unreachable_items()
|
|
inventory = {
|
|
"roots": list(args.roots),
|
|
"excluded_directories": sorted(EXCLUDED_DIRS),
|
|
"excluded_output_dir": relative_path(output_dir, repo_root),
|
|
"items": inventory_entries,
|
|
"unreachable_items": [{"path": item["path"], "reason": item["anomalies"][0]["message"]} for item in unreachable_items],
|
|
"accessible_item_count": len(inventory_entries),
|
|
"item_count": len(inventory_entries) + len(unreachable_items),
|
|
"inventory_hash": canonical_hash({"items": inventory_entries, "unreachable_items": [{"path": item["path"], "reason": item["anomalies"][0]["message"]} for item in unreachable_items]}),
|
|
}
|
|
checkpoint_path = output_dir / "scan-checkpoint.json"
|
|
checkpoint: dict[str, Any] | None = None
|
|
if args.resume and checkpoint_path.is_file():
|
|
try:
|
|
candidate = json.loads(checkpoint_path.read_text(encoding="utf-8"))
|
|
if candidate.get("inventory", {}).get("inventory_hash") == inventory["inventory_hash"] and candidate.get("scanner_version") == SCANNER_VERSION:
|
|
checkpoint = candidate
|
|
except (OSError, json.JSONDecodeError):
|
|
checkpoint = None
|
|
records = {item["path"]: item for item in (checkpoint or {}).get("records", []) if isinstance(item, dict) and item.get("path")}
|
|
started_at = (checkpoint or {}).get("started_at") or utc_now()
|
|
for offset in range(0, len(paths), args.batch_size):
|
|
for path in paths[offset: offset + args.batch_size]:
|
|
rel = relative_path(path, repo_root)
|
|
existing = records.get(rel)
|
|
fingerprint = next(entry for entry in inventory_entries if entry["path"] == rel)
|
|
if existing and existing.get("size_bytes") == fingerprint["size_bytes"] and existing.get("modified_ns") == fingerprint["modified_ns"]:
|
|
continue
|
|
records[rel] = make_item(path, repo_root)
|
|
atomic_json(checkpoint_path, {"schema_version": 1, "scanner_version": SCANNER_VERSION, "started_at": started_at, "inventory": inventory, "cursor": min(offset + args.batch_size, len(paths)), "records": sorted(records.values(), key=lambda item: item["path"])})
|
|
items = [records[relative_path(path, repo_root)] for path in paths if relative_path(path, repo_root) in records]
|
|
items.extend(unreachable_items)
|
|
scan_id = "p3-" + inventory["inventory_hash"][:16]
|
|
report = report_payload(items, inventory, scan_id, started_at=started_at, completed_at=utc_now(), repo_root=repo_root)
|
|
output_dir.mkdir(parents=True, exist_ok=True)
|
|
atomic_json(output_dir / "full-scan-manifest.json", report)
|
|
atomic_json(output_dir / "anomaly-manifest.json", {"schema_version": 1, "scan_id": scan_id, "anomalies": report["anomalies"], "counts": report["anomaly_category_counts"]})
|
|
atomic_json(output_dir / "quarantine-manifest.json", {"schema_version": 1, "scan_id": scan_id, "items": report["quarantine"], "source_files_unchanged": True})
|
|
atomic_json(output_dir / "duplicates-report.json", report["duplicates"] | {"scan_id": scan_id})
|
|
atomic_json(output_dir / "leakage-report.json", report["leakage"] | {"scan_id": scan_id})
|
|
atomic_json(output_dir / "source-freshness-report.json", {"schema_version": 1, "scan_id": scan_id, "items": report["source_freshness"]})
|
|
atomic_json(output_dir / "dataset-summary.json", {"schema_version": 1, "scan_id": scan_id, "datasets": report["dataset_summary"], "anomaly_category_counts": report["anomaly_category_counts"]})
|
|
print(json.dumps({"scan_id": scan_id, "inventory_total": len(items), "reconciliation": report["reconciliation"], "anomaly_count": report["anomaly_count"], "content_hash": report["determinism"]["content_hash"]}, indent=2))
|
|
return 0 if report["reconciliation"]["reconciles"] else 2
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|