#!/usr/bin/env python3 """Reproducible, read-only inventory and quarantine scan for Accuracy Phase 3. The scanner deliberately operates on immutable source files and writes only to the requested evidence directory. It is dependency-light, but uses rasterio and shapely when available for type-specific checks. A checkpoint makes the scan resumable without making source files mutable. """ from __future__ import annotations import argparse import hashlib import json import math import os import re import tempfile from collections import Counter, defaultdict from datetime import datetime, timezone from pathlib import Path from typing import Any, Iterable SCANNER_VERSION = "3.1.0" SCHEMA_VERSION = 1 DEFAULT_ROOTS = ("models", "datasets", "data", "storage", "artifacts", "output") EXCLUDED_DIRS = {".git", "node_modules", ".next", "__pycache__", ".pytest_cache"} SEVERITIES = ("blocker", "critical", "major", "minor", "informational") ACTIONS = ( "accept", "repairable automatically", "requires review", "quarantine", "exclude from training", "exclude from evaluation", "unavailable or unreadable", ) def utc_now() -> str: return datetime.now(timezone.utc).isoformat() def sha256(path: Path) -> str: digest = hashlib.sha256() with path.open("rb") as stream: for chunk in iter(lambda: stream.read(1024 * 1024), b""): digest.update(chunk) return digest.hexdigest() def canonical_hash(value: Any) -> str: return hashlib.sha256( json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode("utf-8") ).hexdigest() def atomic_json(path: Path, payload: Any) -> None: path.parent.mkdir(parents=True, exist_ok=True) fd, temp_name = tempfile.mkstemp(prefix=f".{path.name}.", suffix=".tmp", dir=path.parent) try: with os.fdopen(fd, "w", encoding="utf-8", newline="\n") as stream: json.dump(payload, stream, ensure_ascii=False, indent=2, sort_keys=True) stream.write("\n") Path(temp_name).replace(path) finally: if os.path.exists(temp_name): os.unlink(temp_name) def severity_action(severity: str, action: str, code: str, message: str, *, field: str | None = None) -> dict[str, Any]: if severity not in SEVERITIES or action not in ACTIONS: raise ValueError(f"Unknown severity/action: {severity}/{action}") result: dict[str, Any] = {"code": code, "severity": severity, "action": action, "message": message} if field: result["field"] = field return result def relative_path(path: Path, repo_root: Path) -> str: return path.resolve(strict=False).relative_to(repo_root.resolve()).as_posix() def is_excluded(path: Path, repo_root: Path, evidence_dir: Path) -> bool: rel = relative_path(path, repo_root) if any(part in EXCLUDED_DIRS for part in Path(rel).parts): return True evidence_rel = relative_path(evidence_dir, repo_root) return rel == evidence_rel or rel.startswith(f"{evidence_rel}/") def discover_files(repo_root: Path, roots: Iterable[str], evidence_dir: Path) -> list[Path]: files: list[Path] = [] for root in roots: base = (repo_root / root).resolve(strict=False) if not base.exists(): continue for path in base.rglob("*"): if path.is_file() and not is_excluded(path, repo_root, evidence_dir): files.append(path) return sorted(set(files), key=lambda item: relative_path(item, repo_root)) def unreachable_items(entries: Iterable[tuple[str, str]]) -> list[dict[str, Any]]: """Create explicit scope records without inventing environment availability. Older scanner versions always emitted three hard-coded unreachable items, even when executed inside the production container where the corresponding model and training mounts were available. Callers must now declare only boundaries they have actually established as unavailable; requested roots that do not exist are added automatically by :func:`missing_root_entries`. """ result = [] for path, reason in entries: result.append( { "item_id": hashlib.sha256(path.encode("utf-8")).hexdigest()[:20], "path": path, "status": "unreachable", "read_status": "unavailable", "size_bytes": None, "modified_at": None, "sha256": None, "duplicate_group": None, "near_duplicate_fingerprint": None, "contract": {"key": "geointel.external.unavailable", "version": "1.0.0"}, "source": "unknown", "lineage": None, "schema_conformity": "unavailable", "crs": None, "units": None, "resolution": None, "spatial_coverage": None, "temporal_coverage": None, "freshness": {"status": "unavailable", "reason": reason}, "geometry_validation": None, "empty_content": None, "anomalies": [severity_action("critical", "unavailable or unreadable", "scope.unreachable", reason)], "recommended_action": "unavailable or unreadable", } ) return result def parse_unreachable_scope(value: str) -> tuple[str, str]: path, separator, reason = value.partition("=") if not separator or not path.strip() or not reason.strip(): raise argparse.ArgumentTypeError( "--unreachable-scope must use PATH=CONCRETE_REASON" ) return path.strip(), reason.strip() def missing_root_entries(repo_root: Path, roots: Iterable[str]) -> list[tuple[str, str]]: entries: list[tuple[str, str]] = [] for root in roots: base = (repo_root / root).resolve(strict=False) if not base.exists(): entries.append( ( f"root://{root}", f"Configured scan root is unavailable: {base}", ) ) return entries def infer_source(rel: str, payload: Any = None) -> str: text = rel.lower() if isinstance(payload, dict): text += " " + json.dumps(payload, ensure_ascii=False, sort_keys=True).lower()[:5000] for token, name in (("grb", "GRB"), ("gebouwenregister", "Gebouwenregister"), ("dhmv", "DHMV"), ("sentinel", "Sentinel-2"), ("osm", "OSM"), ("orthofoto", "Orthophoto")): if token in text: return name return "unknown" def contract_for(path: Path, payload: Any = None) -> tuple[str, str]: suffix = path.suffix.lower() text = path.as_posix().lower() if suffix in {".geojson", ".jsonl"} or "vector" in text: return "geointel.vector.geojson", "1.0.0" if suffix in {".tif", ".tiff", ".cog"}: return "geointel.raster.geotiff", "1.0.0" if suffix in {".pt", ".pth", ".onnx", ".safetensors"} or "model" in text and suffix not in {".json", ".md"}: return "geointel.model.pytorch", "1.0.0" if suffix == ".txt" and any(token in text for token in ("label", "yolo", "annotation")): return "geointel.label.yolo", "1.1.0" if suffix == ".db" or suffix in {".db-wal", ".db-shm"}: return "geointel.database.sqlite", "1.0.0" if suffix == ".json" and isinstance(payload, dict) and ("lineage" in payload or "schema_version" in payload): return "geointel.manifest.json", "1.0.0" return "geointel.artifact.generic", "1.0.0" def add_anomaly(item: dict[str, Any], anomaly: dict[str, Any]) -> None: item.setdefault("anomalies", []).append(anomaly) def parse_json(path: Path) -> tuple[Any, str | None]: try: return json.loads(path.read_text(encoding="utf-8-sig")), None except Exception as exc: # noqa: BLE001 - evidence must retain the concrete parser failure return None, f"{type(exc).__name__}: {exc}" def validate_geometry(item: dict[str, Any], payload: Any) -> None: if not isinstance(payload, dict) or payload.get("type") != "FeatureCollection": add_anomaly(item, severity_action("critical", "quarantine", "vector.invalid_schema", "Expected GeoJSON FeatureCollection")) item["schema_conformity"] = "invalid" return features = payload.get("features") if not isinstance(features, list): add_anomaly(item, severity_action("critical", "quarantine", "vector.features_not_list", "GeoJSON features must be a list")) item["schema_conformity"] = "invalid" return item["feature_count"] = len(features) item["empty_content"] = len(features) == 0 if len(features) == 0: add_anomaly(item, severity_action("minor", "requires review", "vector.empty", "GeoJSON contains no features")) try: from shapely.geometry import shape except Exception: item["geometry_validation"] = {"status": "unavailable", "reason": "shapely_not_installed"} return invalid = 0 empty = 0 bounds: list[float] | None = None for index, feature in enumerate(features): geometry = feature.get("geometry") if isinstance(feature, dict) else None try: geom = shape(geometry) if geometry else None if geom is None or geom.is_empty: empty += 1 elif not geom.is_valid: invalid += 1 elif not geom.is_empty: candidate = list(geom.bounds) bounds = candidate if bounds is None else [min(bounds[0], candidate[0]), min(bounds[1], candidate[1]), max(bounds[2], candidate[2]), max(bounds[3], candidate[3])] except Exception as exc: # noqa: BLE001 invalid += 1 if invalid == 1: add_anomaly(item, severity_action("critical", "quarantine", "vector.geometry_parse_error", f"Feature {index}: {exc}")) item["geometry_validation"] = {"status": "ok" if invalid == 0 else "invalid", "invalid_count": invalid, "empty_count": empty} item["spatial_coverage"] = {"bbox": bounds, "crs": "EPSG:4326 (implicit GeoJSON)"} if invalid: add_anomaly(item, severity_action("critical", "quarantine", "vector.invalid_geometry", f"{invalid} invalid geometries")) if bounds and (bounds[0] < -180 or bounds[2] > 180 or bounds[1] < -90 or bounds[3] > 90): add_anomaly(item, severity_action("critical", "quarantine", "vector.bbox_out_of_range", "EPSG:4326 coordinates exceed valid bounds", field="bbox")) def validate_raster(item: dict[str, Any], path: Path) -> None: try: import rasterio import numpy as np with rasterio.open(path) as dataset: item["raster"] = { "width": dataset.width, "height": dataset.height, "bands": dataset.count, "dtype": list(dataset.dtypes), "crs": dataset.crs.to_string() if dataset.crs else None, "resolution": [float(dataset.res[0]), float(dataset.res[1])], "bounds": [float(v) for v in dataset.bounds], "nodata": dataset.nodata, } item["crs"] = item["raster"]["crs"] item["resolution"] = item["raster"]["resolution"] item["spatial_coverage"] = {"bbox": item["raster"]["bounds"], "crs": item["crs"]} item["empty_content"] = dataset.width == 0 or dataset.height == 0 if not dataset.crs: add_anomaly(item, severity_action("critical", "quarantine", "raster.missing_crs", "Raster has no CRS")) if dataset.width <= 0 or dataset.height <= 0: add_anomaly(item, severity_action("critical", "quarantine", "raster.empty_dimensions", "Raster has invalid dimensions")) sample = dataset.read(1, masked=True, out_shape=(1, min(dataset.height, 256), min(dataset.width, 256))) item["raster"]["sample_valid_fraction"] = float(np.ma.count(sample) / max(sample.size, 1)) if item["raster"]["sample_valid_fraction"] == 0: add_anomaly(item, severity_action("major", "quarantine", "raster.all_nodata", "Raster sample contains no valid pixels")) if any(not math.isfinite(value) for value in item["raster"]["resolution"]) or min(item["raster"]["resolution"]) <= 0: add_anomaly(item, severity_action("critical", "quarantine", "raster.invalid_resolution", "Raster resolution is non-positive or non-finite")) except Exception as exc: # noqa: BLE001 item["read_status"] = "unreadable" add_anomaly(item, severity_action("critical", "unavailable or unreadable", "raster.unreadable", f"{type(exc).__name__}: {exc}")) def validate_manifest(item: dict[str, Any], payload: Any, repo_root: Path) -> None: if not isinstance(payload, dict): return if "crs" in payload: item["crs"] = payload.get("crs") if "bounds" in payload: item["spatial_coverage"] = {"bbox": payload.get("bounds"), "crs": payload.get("crs")} bounds = payload.get("bounds") if isinstance(bounds, list) and len(bounds) == 4 and all(float(v) == 0 for v in bounds): add_anomaly(item, severity_action("major", "quarantine", "manifest.zero_bbox", "Manifest declares a zero-area bounding box", field="bounds")) if "tile_paths" in payload and isinstance(payload["tile_paths"], list): missing = [] for raw in payload["tile_paths"]: candidate = Path(str(raw)) if not candidate.is_absolute(): candidate = repo_root / candidate if not candidate.exists(): missing.append(str(raw)) item["lineage"] = {"source_dataset_id": payload.get("source_dataset_id"), "source_raster_id": payload.get("source_raster_id")} if missing: add_anomaly(item, severity_action("critical", "quarantine", "manifest.missing_tile", f"{len(missing)} tile paths are absent")) item["missing_tile_paths"] = missing def validate_label(item: dict[str, Any], path: Path) -> None: try: lines = [line.strip() for line in path.read_text(encoding="utf-8").splitlines() if line.strip()] except Exception as exc: # noqa: BLE001 add_anomaly(item, severity_action("critical", "unavailable or unreadable", "label.unreadable", str(exc))) return invalid = 0 for line in lines: parts = line.split() try: if len(parts) != 5 or int(parts[0]) < 0 or any(not 0 <= float(value) <= 1 for value in parts[1:]): invalid += 1 except ValueError: invalid += 1 item["label"] = {"line_count": len(lines), "invalid_line_count": invalid} if invalid: add_anomaly(item, severity_action("critical", "quarantine", "label.invalid_yolo", f"{invalid} label lines violate YOLO normalized schema")) if not lines: item["empty_content"] = True add_anomaly(item, severity_action("minor", "requires review", "label.empty", "Empty label file requires explicit background review evidence")) def json_metadata(item: dict[str, Any], payload: Any) -> None: if not isinstance(payload, dict): return observed = next((payload.get(key) for key in ("observed_at", "created_at", "updated_at", "timestamp", "fetched_at") if payload.get(key)), None) if observed: item["freshness"] = {"observed_at": observed, "method": "declared_metadata", "status": "declared"} lineage_keys = [key for key in payload if "lineage" in key.lower() or key.startswith("source_") or key in {"parent_id", "parent_dataset_id"}] if lineage_keys: item["lineage"] = {key: payload.get(key) for key in lineage_keys} def normalized_fingerprint(path: Path, payload: Any) -> str | None: if payload is not None: return canonical_hash(payload) if path.suffix.lower() not in {".png", ".jpg", ".jpeg", ".webp"}: return None try: from PIL import Image image = Image.open(path).convert("L").resize((16, 16)) pixels = list(image.getdata()) average = sum(pixels) / max(len(pixels), 1) return "image:" + "".join("1" if pixel >= average else "0" for pixel in pixels) except Exception: return None def make_item(path: Path, repo_root: Path) -> dict[str, Any]: rel = relative_path(path, repo_root) stat = path.stat() item: dict[str, Any] = { "item_id": hashlib.sha256(rel.encode("utf-8")).hexdigest()[:20], "path": rel, "status": "examined", "read_status": "readable", "size_bytes": stat.st_size, "modified_at": datetime.fromtimestamp(stat.st_mtime, tz=timezone.utc).isoformat(), "sha256": None, "duplicate_group": None, "near_duplicate_fingerprint": None, "contract": None, "source": infer_source(rel), "lineage": None, "schema_conformity": "not_applicable", "crs": None, "units": None, "resolution": None, "spatial_coverage": None, "temporal_coverage": None, "freshness": {"status": "unknown", "reason": "no_declared_source_timestamp"}, "geometry_validation": None, "empty_content": stat.st_size == 0, "anomalies": [], } try: item["sha256"] = sha256(path) except Exception as exc: # noqa: BLE001 item["status"] = "unreachable" item["read_status"] = "unreadable" add_anomaly(item, severity_action("critical", "unavailable or unreadable", "file.unreadable", f"{type(exc).__name__}: {exc}")) return item payload = None parse_error = None if path.suffix.lower() in {".json", ".geojson"} or path.name.lower().endswith(".jsonl"): payload, parse_error = parse_json(path) if parse_error: item["schema_conformity"] = "invalid" add_anomaly(item, severity_action("critical", "quarantine", "json.invalid", parse_error)) else: json_metadata(item, payload) contract_key, contract_version = contract_for(path, payload) item["contract"] = {"key": contract_key, "version": contract_version} item["source"] = infer_source(rel, payload) if path.suffix.lower() == ".geojson" and payload is not None: item["schema_conformity"] = "valid" validate_geometry(item, payload) elif path.suffix.lower() in {".tif", ".tiff", ".cog"}: validate_raster(item, path) item["schema_conformity"] = "valid" if item["read_status"] == "readable" else "invalid" elif path.name.lower() == "manifest.json" and payload is not None: item["schema_conformity"] = "valid" if isinstance(payload, dict) else "invalid" validate_manifest(item, payload, repo_root) elif contract_key == "geointel.label.yolo" and path.suffix.lower() == ".txt": item["schema_conformity"] = "valid" validate_label(item, path) if item["source"] == "OSM": item["ground_truth_eligible"] = False add_anomaly(item, severity_action("informational", "requires review", "source.osm_not_ground_truth", "OSM is corroborative/contextual and never automatic ground truth")) if item["lineage"] is None and ("derived" in rel.lower() or "result" in rel.lower() or "calibration" in rel.lower()): add_anomaly(item, severity_action("major", "quarantine", "lineage.missing", "Derived-looking artifact has no explicit lineage")) if item["freshness"].get("status") == "unknown" and item["source"] not in {"unknown", "Orthophoto"}: add_anomaly(item, severity_action("minor", "requires review", "freshness.missing", "Source-labelled item has no declared observation timestamp")) item["near_duplicate_fingerprint"] = normalized_fingerprint(path, payload) if item["anomalies"]: item["recommended_action"] = sorted(item["anomalies"], key=lambda a: SEVERITIES.index(a["severity"]))[0]["action"] else: item["recommended_action"] = "accept" return item def detect_duplicates(items: list[dict[str, Any]]) -> dict[str, Any]: by_hash: dict[str, list[str]] = defaultdict(list) by_fingerprint: dict[str, list[str]] = defaultdict(list) for item in items: if item.get("sha256"): by_hash[item["sha256"]].append(item["path"]) if item.get("near_duplicate_fingerprint"): by_fingerprint[item["near_duplicate_fingerprint"]].append(item["path"]) exact = {key: sorted(paths) for key, paths in by_hash.items() if len(paths) > 1} near = {key: sorted(paths) for key, paths in by_fingerprint.items() if len(paths) > 1} for group, paths in exact.items(): for item in items: if item["path"] in paths: item["duplicate_group"] = group return {"schema_version": 1, "exact_duplicate_groups": exact, "near_duplicate_groups": near, "method": "sha256_and_normalized_payload_or_image_ahash"} def detect_leakage(items: list[dict[str, Any]], repo_root: Path | None = None) -> dict[str, Any]: split_by_hash: dict[str, set[str]] = defaultdict(set) split_by_bbox: dict[str, list[tuple[str, list[float], str]]] = defaultdict(list) for item in items: text = item["path"].lower() split = next((candidate for candidate in ("train", "val", "validation", "calibration", "test", "background-test") if re.search(rf"(?:^|[/_.-]){re.escape(candidate)}(?:[/_.-]|$)", text)), None) if not split: continue if item.get("sha256"): split_by_hash[item["sha256"]].add(split) bbox = (item.get("spatial_coverage") or {}).get("bbox") if isinstance(item.get("spatial_coverage"), dict) else None if isinstance(bbox, list) and len(bbox) == 4: try: split_by_bbox[split].append((item["path"], [float(value) for value in bbox], split)) except (TypeError, ValueError): pass cross_hash = {key: sorted(value) for key, value in split_by_hash.items() if len(value) > 1} source_audits = [item["path"] for item in items if item["path"].endswith("spatial-leakage-audit.json")] prior_quality: dict[str, Any] | None = None prior_attention = False if repo_root is not None: status_path = repo_root / "docs/accuracy-program/status.json" try: status = json.loads(status_path.read_text(encoding="utf-8")) v56 = ((status.get("ml_data") or {}).get("v56") or {}) prior_quality = { "path": relative_path(status_path, repo_root), "cross_split_pairs_below_2000_m": v56.get("cross_split_pairs_below_2000_m"), "split_independence_proven": v56.get("split_independence_proven"), "exact_cross_split_raster_hash_duplicates": v56.get("exact_cross_split_raster_hash_duplicates"), } prior_attention = (prior_quality.get("cross_split_pairs_below_2000_m") or 0) > 0 or prior_quality.get("split_independence_proven") is False except (OSError, json.JSONDecodeError): prior_quality = None return { "schema_version": 1, "status": "attention" if cross_hash or source_audits or prior_attention else "no_detected_overlap", "same_checksum_across_splits": cross_hash, "spatial_overlap_checks": "not_proven_without_AOI_split_geometry", "source_spatial_audit_files": source_audits, "prior_quality_inventory": prior_quality, "limitations": ["Filename-derived split tokens are conservative; AOI independence requires authoritative split geometry."], } def report_payload(items: list[dict[str, Any]], inventory: dict[str, Any], scan_id: str, *, started_at: str, completed_at: str, repo_root: Path | None = None) -> dict[str, Any]: anomalies = [dict({"path": item["path"], "item_id": item["item_id"]}, **anomaly) for item in items for anomaly in item.get("anomalies", [])] quarantine = [ {"path": item["path"], "item_id": item["item_id"], "recommended_action": item.get("recommended_action"), "anomalies": item.get("anomalies", [])} for item in items if item.get("recommended_action") in {"quarantine", "exclude from training", "exclude from evaluation", "unavailable or unreadable"} ] duplicates = detect_duplicates(items) leakage = detect_leakage(items, repo_root) category_counts = Counter(anomaly["code"] for anomaly in anomalies) dataset_summary: dict[str, dict[str, Any]] = {} for item in items: key = item["contract"]["key"] if item.get("contract") else "unknown" block = dataset_summary.setdefault(key, {"item_count": 0, "anomaly_count": 0, "paths": [], "severity_counts": Counter()}) block["item_count"] += 1 block["paths"].append(item["path"]) block["anomaly_count"] += len(item.get("anomalies", [])) block["severity_counts"].update(anomaly["severity"] for anomaly in item.get("anomalies", [])) for block in dataset_summary.values(): block["paths"] = sorted(block["paths"]) block["severity_counts"] = dict(sorted(block["severity_counts"].items())) source_freshness = Counter((item.get("source", "unknown"), (item.get("freshness") or {}).get("status", "unknown")) for item in items) counts = Counter(item.get("status", "examined") for item in items) reconciliation = {"examined": counts.get("examined", 0), "skipped": counts.get("skipped", 0), "unreachable": counts.get("unreachable", 0), "inventory_total": len(items), "reconciles": sum(counts.get(key, 0) for key in ("examined", "skipped", "unreachable")) == len(items)} return { "schema_version": SCHEMA_VERSION, "scanner_version": SCANNER_VERSION, "scan_id": scan_id, "started_at": started_at, "completed_at": completed_at, "inventory": inventory, "reconciliation": reconciliation, "items": sorted(items, key=lambda item: item["path"]), "anomaly_count": len(anomalies), "anomalies": sorted(anomalies, key=lambda item: (item["path"], item["code"])), "quarantine": sorted(quarantine, key=lambda item: item["path"]), "duplicates": duplicates, "leakage": leakage, "dataset_summary": {key: dataset_summary[key] for key in sorted(dataset_summary)}, "source_freshness": {f"{source}|{status}": count for (source, status), count in sorted(source_freshness.items())}, "anomaly_category_counts": dict(sorted(category_counts.items())), "grb_consistency": {"status": "unavailable", "reason": "No authoritative GRB snapshot was accessible in the configured local roots; no derived result was marked as GRB ground truth."}, "determinism": {"content_hash": canonical_hash({"inventory": inventory, "items": sorted(items, key=lambda item: item["path"]), "anomalies": sorted(anomalies, key=lambda item: (item["path"], item["code"]))}), "timestamps_excluded_from_content_hash": True}, } def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--repo-root", type=Path, default=Path(__file__).resolve().parents[1]) parser.add_argument("--output-dir", type=Path, default=None) parser.add_argument("--batch-size", type=int, default=50) parser.add_argument("--resume", action="store_true") parser.add_argument("--roots", nargs="+", default=list(DEFAULT_ROOTS)) parser.add_argument( "--unreachable-scope", action="append", default=[], type=parse_unreachable_scope, metavar="PATH=CONCRETE_REASON", help=( "Explicit non-filesystem or externally mounted boundary that was " "verified as unavailable. May be repeated." ), ) return parser.parse_args() def main() -> int: args = parse_args() repo_root = args.repo_root.resolve() output_dir = (args.output_dir or repo_root / "artifacts/evidence/accuracy/P3").resolve() if args.batch_size <= 0: raise SystemExit("--batch-size must be positive") paths = discover_files(repo_root, args.roots, output_dir) inventory_entries = [{"path": relative_path(path, repo_root), "size_bytes": path.stat().st_size, "modified_ns": path.stat().st_mtime_ns} for path in paths] unavailable = unreachable_items( [*missing_root_entries(repo_root, args.roots), *args.unreachable_scope] ) inventory = { "roots": list(args.roots), "excluded_directories": sorted(EXCLUDED_DIRS), "excluded_output_dir": relative_path(output_dir, repo_root), "items": inventory_entries, "unreachable_items": [{"path": item["path"], "reason": item["anomalies"][0]["message"]} for item in unavailable], "accessible_item_count": len(inventory_entries), "item_count": len(inventory_entries) + len(unavailable), "inventory_hash": canonical_hash({"items": inventory_entries, "unreachable_items": [{"path": item["path"], "reason": item["anomalies"][0]["message"]} for item in unavailable]}), } checkpoint_path = output_dir / "scan-checkpoint.json" checkpoint: dict[str, Any] | None = None if args.resume and checkpoint_path.is_file(): try: candidate = json.loads(checkpoint_path.read_text(encoding="utf-8")) if candidate.get("inventory", {}).get("inventory_hash") == inventory["inventory_hash"] and candidate.get("scanner_version") == SCANNER_VERSION: checkpoint = candidate except (OSError, json.JSONDecodeError): checkpoint = None records = {item["path"]: item for item in (checkpoint or {}).get("records", []) if isinstance(item, dict) and item.get("path")} started_at = (checkpoint or {}).get("started_at") or utc_now() for offset in range(0, len(paths), args.batch_size): for path in paths[offset: offset + args.batch_size]: rel = relative_path(path, repo_root) existing = records.get(rel) fingerprint = next(entry for entry in inventory_entries if entry["path"] == rel) if existing and existing.get("size_bytes") == fingerprint["size_bytes"] and existing.get("modified_ns") == fingerprint["modified_ns"]: continue records[rel] = make_item(path, repo_root) atomic_json(checkpoint_path, {"schema_version": 1, "scanner_version": SCANNER_VERSION, "started_at": started_at, "inventory": inventory, "cursor": min(offset + args.batch_size, len(paths)), "records": sorted(records.values(), key=lambda item: item["path"])}) items = [records[relative_path(path, repo_root)] for path in paths if relative_path(path, repo_root) in records] items.extend(unavailable) scan_id = "p3-" + inventory["inventory_hash"][:16] report = report_payload(items, inventory, scan_id, started_at=started_at, completed_at=utc_now(), repo_root=repo_root) output_dir.mkdir(parents=True, exist_ok=True) atomic_json(output_dir / "full-scan-manifest.json", report) atomic_json(output_dir / "anomaly-manifest.json", {"schema_version": 1, "scan_id": scan_id, "anomalies": report["anomalies"], "counts": report["anomaly_category_counts"]}) atomic_json(output_dir / "quarantine-manifest.json", {"schema_version": 1, "scan_id": scan_id, "items": report["quarantine"], "source_files_unchanged": True}) atomic_json(output_dir / "duplicates-report.json", report["duplicates"] | {"scan_id": scan_id}) atomic_json(output_dir / "leakage-report.json", report["leakage"] | {"scan_id": scan_id}) atomic_json(output_dir / "source-freshness-report.json", {"schema_version": 1, "scan_id": scan_id, "items": report["source_freshness"]}) atomic_json(output_dir / "dataset-summary.json", {"schema_version": 1, "scan_id": scan_id, "datasets": report["dataset_summary"], "anomaly_category_counts": report["anomaly_category_counts"]}) print(json.dumps({"scan_id": scan_id, "inventory_total": len(items), "reconciliation": report["reconciliation"], "anomaly_count": report["anomaly_count"], "content_hash": report["determinism"]["content_hash"]}, indent=2)) return 0 if report["reconciliation"]["reconciles"] else 2 if __name__ == "__main__": raise SystemExit(main())