audit: establish accuracy phase 1 baseline
This commit is contained in:
Jens
@@ -0,0 +1,159 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
from datetime import datetime, timezone
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
from typing import Any
|
||||
|
||||
|
||||
REPOSITORY_ROOT = Path(__file__).resolve().parents[1]
|
||||
DEFAULT_EVIDENCE_ROOT = REPOSITORY_ROOT / "artifacts" / "evidence" / "accuracy" / "P1"
|
||||
DEFAULT_OUTPUT = DEFAULT_EVIDENCE_ROOT / "evidence-manifest.json"
|
||||
|
||||
PROGRAM_PATHS = (
|
||||
"docs/accuracy-program/00-execution-contract.md",
|
||||
"docs/accuracy-program/01-system-inventory.md",
|
||||
"docs/accuracy-program/02-data-lineage.md",
|
||||
"docs/accuracy-program/03-baseline-and-gaps.md",
|
||||
"docs/accuracy-program/04-risk-register.md",
|
||||
"docs/accuracy-program/05-metric-framework.md",
|
||||
"docs/accuracy-program/06-implementation-roadmap.md",
|
||||
"docs/accuracy-program/status.json",
|
||||
"scripts/build_accuracy_phase1_evidence_manifest.py",
|
||||
"scripts/collect_accuracy_phase1_inference_smoke.py",
|
||||
"scripts/collect_accuracy_phase1_ml_lineage.py",
|
||||
"scripts/collect_accuracy_phase1_runtime.py",
|
||||
"scripts/reproduce_accuracy_phase1_findings.py",
|
||||
"scripts/run_accuracy_phase1_baseline.py",
|
||||
"scripts/verify_accuracy_phase1_evidence.py",
|
||||
"tests/test_accuracy_phase1_baseline.py",
|
||||
)
|
||||
|
||||
|
||||
def sha256_file(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as handle:
|
||||
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def role_for(path: Path) -> str:
|
||||
name = path.name.lower()
|
||||
if name.endswith(".junit.xml"):
|
||||
return "test_report"
|
||||
if name.endswith(".sql"):
|
||||
return "migration_evidence"
|
||||
if "runtime" in name or "gpu-inference" in name:
|
||||
return "runtime_evidence"
|
||||
if "lineage" in name:
|
||||
return "lineage_evidence"
|
||||
if "reproduction" in name:
|
||||
return "defect_reproduction"
|
||||
if "ruff" in name or "lint" in name:
|
||||
return "lint_evidence"
|
||||
if "test" in name or "vitest" in name or "golden-qa" in name:
|
||||
return "test_evidence"
|
||||
if path.suffix.lower() == ".json":
|
||||
return "structured_inventory"
|
||||
return "execution_log"
|
||||
|
||||
|
||||
def record(path: Path, *, displayed_path: str, role: str) -> dict[str, Any]:
|
||||
return {
|
||||
"path": displayed_path,
|
||||
"role": role,
|
||||
"size_bytes": path.stat().st_size,
|
||||
"sha256": sha256_file(path),
|
||||
}
|
||||
|
||||
|
||||
def git_head() -> str | None:
|
||||
result = subprocess.run(
|
||||
["git", "rev-parse", "HEAD"],
|
||||
cwd=REPOSITORY_ROOT,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
)
|
||||
return result.stdout.strip() if result.returncode == 0 else None
|
||||
|
||||
|
||||
def build_manifest(evidence_root: Path, output: Path) -> dict[str, Any]:
|
||||
if not evidence_root.is_dir():
|
||||
raise FileNotFoundError(f"Evidence root does not exist: {evidence_root}")
|
||||
|
||||
evidence_files = [
|
||||
path
|
||||
for path in evidence_root.rglob("*")
|
||||
if path.is_file() and path.resolve() != output.resolve()
|
||||
]
|
||||
evidence_records = [
|
||||
record(
|
||||
path,
|
||||
displayed_path=path.relative_to(REPOSITORY_ROOT).as_posix(),
|
||||
role=role_for(path),
|
||||
)
|
||||
for path in sorted(evidence_files)
|
||||
]
|
||||
|
||||
program_records = []
|
||||
for relative in PROGRAM_PATHS:
|
||||
path = REPOSITORY_ROOT / relative
|
||||
if not path.is_file():
|
||||
raise FileNotFoundError(f"Required Phase-1 program file is missing: {relative}")
|
||||
program_records.append(record(path, displayed_path=relative, role="phase1_program"))
|
||||
|
||||
return {
|
||||
"schema_version": 1,
|
||||
"generated_at": datetime.now(timezone.utc).isoformat(),
|
||||
"audited_repository_head": git_head(),
|
||||
"claim_boundary": (
|
||||
"This manifest proves retained-file identity and completeness. It does "
|
||||
"not establish model accuracy, human label acceptance, split independence "
|
||||
"or release readiness."
|
||||
),
|
||||
"evidence_root": evidence_root.relative_to(REPOSITORY_ROOT).as_posix(),
|
||||
"evidence_file_count": len(evidence_records),
|
||||
"evidence_total_bytes": sum(item["size_bytes"] for item in evidence_records),
|
||||
"evidence_files": evidence_records,
|
||||
"program_file_count": len(program_records),
|
||||
"program_files": program_records,
|
||||
}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Build the immutable GeoIntel Accuracy P1 evidence manifest.")
|
||||
parser.add_argument("--evidence-root", type=Path, default=DEFAULT_EVIDENCE_ROOT)
|
||||
parser.add_argument("--output", type=Path, default=DEFAULT_OUTPUT)
|
||||
args = parser.parse_args()
|
||||
|
||||
evidence_root = args.evidence_root.expanduser().resolve()
|
||||
output = args.output.expanduser().resolve()
|
||||
if output.exists():
|
||||
parser.error(f"refusing to overwrite existing evidence manifest: {output}")
|
||||
if output.parent != evidence_root:
|
||||
parser.error("--output must be directly inside --evidence-root")
|
||||
|
||||
payload = build_manifest(evidence_root, output)
|
||||
output.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"status": "created",
|
||||
"output": str(output),
|
||||
"evidence_file_count": payload["evidence_file_count"],
|
||||
"program_file_count": payload["program_file_count"],
|
||||
},
|
||||
indent=2,
|
||||
sort_keys=True,
|
||||
)
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,181 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
from collections import Counter
|
||||
from datetime import datetime, timezone
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
import random
|
||||
import sys
|
||||
from time import perf_counter
|
||||
from typing import Any
|
||||
|
||||
|
||||
REPOSITORY_ROOT = Path(__file__).resolve().parents[1]
|
||||
BACKEND_ROOT = REPOSITORY_ROOT / "backend"
|
||||
if str(BACKEND_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(BACKEND_ROOT))
|
||||
|
||||
from app.core.config import Settings # noqa: E402
|
||||
from app.services.yolo_adapter import YoloDetectionAdapter # noqa: E402
|
||||
|
||||
|
||||
def _sha256(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as handle:
|
||||
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def _raster_metadata(path: Path) -> dict[str, Any]:
|
||||
try:
|
||||
import rasterio
|
||||
except ImportError:
|
||||
return {"available": False, "reason": "rasterio_not_installed"}
|
||||
|
||||
with rasterio.open(path) as dataset:
|
||||
return {
|
||||
"available": True,
|
||||
"bounds": [float(value) for value in dataset.bounds],
|
||||
"count": int(dataset.count),
|
||||
"crs": str(dataset.crs) if dataset.crs else None,
|
||||
"dtypes": list(dataset.dtypes),
|
||||
"height": int(dataset.height),
|
||||
"nodata": dataset.nodata,
|
||||
"transform": [float(value) for value in dataset.transform],
|
||||
"width": int(dataset.width),
|
||||
}
|
||||
|
||||
|
||||
def _summarize_detections(detections: list[dict[str, Any]]) -> dict[str, Any]:
|
||||
confidences = [float(item["confidence"]) for item in detections]
|
||||
class_counts = Counter(str(item["class_name"]) for item in detections)
|
||||
return {
|
||||
"count": len(detections),
|
||||
"class_counts": dict(sorted(class_counts.items())),
|
||||
"confidence": {
|
||||
"minimum": min(confidences) if confidences else None,
|
||||
"maximum": max(confidences) if confidences else None,
|
||||
"mean": sum(confidences) / len(confidences) if confidences else None,
|
||||
},
|
||||
"sample": detections[:10],
|
||||
}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(
|
||||
description=(
|
||||
"Run one read-only production-adapter inference and emit forensic JSON. "
|
||||
"This proves runtime execution only; it does not establish model accuracy."
|
||||
)
|
||||
)
|
||||
parser.add_argument("--model-path", required=True)
|
||||
parser.add_argument("--tile-path", required=True)
|
||||
parser.add_argument("--manifest-path")
|
||||
parser.add_argument("--confidence", type=float, default=0.5)
|
||||
parser.add_argument("--image-size", type=int, default=640)
|
||||
parser.add_argument("--max-detections", type=int, default=1000)
|
||||
parser.add_argument("--device", default="cuda:0")
|
||||
parser.add_argument("--seed", type=int, default=20260801)
|
||||
args = parser.parse_args()
|
||||
|
||||
model_path = Path(args.model_path).expanduser().resolve()
|
||||
tile_path = Path(args.tile_path).expanduser().resolve()
|
||||
manifest_path = Path(args.manifest_path).expanduser().resolve() if args.manifest_path else None
|
||||
for label, path in (("model", model_path), ("tile", tile_path)):
|
||||
if not path.is_file():
|
||||
parser.error(f"{label} path is not an existing file: {path}")
|
||||
if manifest_path is not None and not manifest_path.is_file():
|
||||
parser.error(f"manifest path is not an existing file: {manifest_path}")
|
||||
if not 0.0 <= args.confidence <= 1.0:
|
||||
parser.error("--confidence must be between 0 and 1")
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
import ultralytics
|
||||
|
||||
random.seed(args.seed)
|
||||
np.random.seed(args.seed)
|
||||
torch.manual_seed(args.seed)
|
||||
if torch.cuda.is_available():
|
||||
torch.cuda.manual_seed_all(args.seed)
|
||||
torch.cuda.reset_peak_memory_stats()
|
||||
torch.use_deterministic_algorithms(True, warn_only=True)
|
||||
|
||||
settings = Settings().model_copy(
|
||||
update={
|
||||
"yolo_enabled": True,
|
||||
"yolo_model_path": str(model_path),
|
||||
"yolo_device": args.device,
|
||||
"yolo_require_cuda": True,
|
||||
"yolo_image_size": args.image_size,
|
||||
"yolo_max_detections": args.max_detections,
|
||||
}
|
||||
)
|
||||
adapter = YoloDetectionAdapter(settings)
|
||||
adapter.validate_runtime()
|
||||
|
||||
started = perf_counter()
|
||||
model = adapter.load_model(model_path)
|
||||
model_loaded = perf_counter()
|
||||
detections = adapter.predict_tile(model, tile_path, args.confidence)
|
||||
if torch.cuda.is_available():
|
||||
torch.cuda.synchronize()
|
||||
finished = perf_counter()
|
||||
|
||||
device_index = torch.cuda.current_device() if torch.cuda.is_available() else None
|
||||
payload = {
|
||||
"schema_version": 1,
|
||||
"captured_at": datetime.now(timezone.utc).isoformat(),
|
||||
"status": "passed",
|
||||
"claim_boundary": (
|
||||
"One production-adapter inference completed on one existing tile. "
|
||||
"No accuracy, calibration, geographic-generalization, or release claim follows from this smoke."
|
||||
),
|
||||
"read_only": True,
|
||||
"configuration": {
|
||||
"confidence": args.confidence,
|
||||
"device": args.device,
|
||||
"image_size": args.image_size,
|
||||
"max_detections": args.max_detections,
|
||||
"seed": args.seed,
|
||||
"deterministic_algorithms": True,
|
||||
},
|
||||
"model": {
|
||||
"path": str(model_path),
|
||||
"sha256": _sha256(model_path),
|
||||
"size_bytes": model_path.stat().st_size,
|
||||
},
|
||||
"input": {
|
||||
"tile_path": str(tile_path),
|
||||
"tile_sha256": _sha256(tile_path),
|
||||
"tile_size_bytes": tile_path.stat().st_size,
|
||||
"manifest_path": str(manifest_path) if manifest_path else None,
|
||||
"manifest_sha256": _sha256(manifest_path) if manifest_path else None,
|
||||
"raster": _raster_metadata(tile_path),
|
||||
},
|
||||
"runtime": {
|
||||
"python": sys.version,
|
||||
"torch": torch.__version__,
|
||||
"ultralytics": ultralytics.__version__,
|
||||
"cuda_available": torch.cuda.is_available(),
|
||||
"cuda_runtime": torch.version.cuda,
|
||||
"cuda_device_index": device_index,
|
||||
"cuda_device_name": torch.cuda.get_device_name(device_index) if device_index is not None else None,
|
||||
"cuda_peak_memory_bytes": torch.cuda.max_memory_allocated() if torch.cuda.is_available() else None,
|
||||
},
|
||||
"timing_seconds": {
|
||||
"model_load": model_loaded - started,
|
||||
"inference": finished - model_loaded,
|
||||
"total": finished - started,
|
||||
},
|
||||
"output": _summarize_detections(detections),
|
||||
}
|
||||
print(json.dumps(payload, indent=2, sort_keys=True))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,364 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
from collections import Counter, defaultdict
|
||||
from datetime import datetime, timezone
|
||||
from itertools import combinations
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
def sha256_file(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as handle:
|
||||
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def artifact_record(path: Path, *, include_hash: bool = True) -> dict[str, Any]:
|
||||
stat = path.stat()
|
||||
return {
|
||||
"path": str(path),
|
||||
"size_bytes": stat.st_size,
|
||||
"modified_at": datetime.fromtimestamp(stat.st_mtime, timezone.utc).isoformat(),
|
||||
"sha256": sha256_file(path) if include_hash else None,
|
||||
}
|
||||
|
||||
|
||||
def manifest_summary(path: Path) -> dict[str, Any]:
|
||||
payload = json.loads(path.read_text(encoding="utf-8"))
|
||||
samples = payload.get("samples") if isinstance(payload.get("samples"), list) else []
|
||||
region_split = Counter(
|
||||
f"{sample.get('region')}/{sample.get('split')}" for sample in samples
|
||||
)
|
||||
contexts = Counter(str(sample.get("context")) for sample in samples)
|
||||
roles = Counter(str(sample.get("sample_role")) for sample in samples)
|
||||
background_samples = [
|
||||
{
|
||||
"sample_slug": sample.get("sample_slug"),
|
||||
"region": sample.get("region"),
|
||||
"reference_feature_count": sample.get("reference_feature_count"),
|
||||
"require_empty": sample.get("require_empty"),
|
||||
}
|
||||
for sample in samples
|
||||
if sample.get("split") == "background-test"
|
||||
]
|
||||
pure_empty = [
|
||||
sample
|
||||
for sample in background_samples
|
||||
if sample.get("require_empty") is True
|
||||
and int(sample.get("reference_feature_count") or 0) == 0
|
||||
]
|
||||
|
||||
cross_split_raster_hashes: dict[str, set[str]] = defaultdict(set)
|
||||
cross_split_dataset_ids: dict[str, set[str]] = defaultdict(set)
|
||||
for sample in samples:
|
||||
split = str(sample.get("split"))
|
||||
if sample.get("raster_sha256"):
|
||||
cross_split_raster_hashes[str(sample["raster_sha256"])].add(split)
|
||||
if sample.get("raster_dataset_id"):
|
||||
cross_split_dataset_ids[str(sample["raster_dataset_id"])].add(split)
|
||||
|
||||
spatial = cross_split_spatial_summary(samples)
|
||||
perceptual = cross_split_dhash_summary(samples)
|
||||
return {
|
||||
**artifact_record(path),
|
||||
"schema_version": payload.get("schema_version"),
|
||||
"dataset_version": payload.get("dataset_version"),
|
||||
"immutable": payload.get("immutable"),
|
||||
"sample_count": len(samples),
|
||||
"region_split_counts": dict(sorted(region_split.items())),
|
||||
"context_counts": dict(sorted(contexts.items())),
|
||||
"sample_role_counts": dict(sorted(roles.items())),
|
||||
"background_test_samples": background_samples,
|
||||
"pure_empty_background_count": len(pure_empty),
|
||||
"pure_empty_background_by_region": dict(
|
||||
sorted(Counter(str(item["region"]) for item in pure_empty).items())
|
||||
),
|
||||
"exact_cross_split_raster_hash_duplicate_count": sum(
|
||||
1 for splits in cross_split_raster_hashes.values() if len(splits) > 1
|
||||
),
|
||||
"cross_split_raster_dataset_id_duplicate_count": sum(
|
||||
1 for splits in cross_split_dataset_ids.values() if len(splits) > 1
|
||||
),
|
||||
"spatial_independence": spatial,
|
||||
"perceptual_duplicate_screen": perceptual,
|
||||
}
|
||||
|
||||
|
||||
def cross_split_spatial_summary(samples: list[dict[str, Any]]) -> dict[str, Any]:
|
||||
try:
|
||||
from pyproj import Transformer
|
||||
from shapely.geometry import box
|
||||
from shapely.ops import transform
|
||||
except ImportError:
|
||||
return {"available": False, "reason": "geospatial_dependencies_unavailable"}
|
||||
|
||||
transformer = Transformer.from_crs(4326, 31370, always_xy=True)
|
||||
records: list[tuple[str, str, Any]] = []
|
||||
for sample in samples:
|
||||
bbox_values = sample.get("bbox_epsg4326")
|
||||
if not isinstance(bbox_values, list) or len(bbox_values) != 4:
|
||||
continue
|
||||
geometry = transform(transformer.transform, box(*[float(value) for value in bbox_values]))
|
||||
records.append((str(sample.get("sample_slug")), str(sample.get("split")), geometry))
|
||||
|
||||
distances: list[tuple[float, str, str, str, str]] = []
|
||||
for left, right in combinations(records, 2):
|
||||
if left[1] == right[1]:
|
||||
continue
|
||||
distance = float(left[2].distance(right[2]))
|
||||
distances.append((distance, left[0], left[1], right[0], right[1]))
|
||||
distances.sort(key=lambda item: item[0])
|
||||
return {
|
||||
"available": True,
|
||||
"crs": "EPSG:31370",
|
||||
"cross_split_pair_count": len(distances),
|
||||
"minimum_distance_m": distances[0][0] if distances else None,
|
||||
"pairs_below_64_m": sum(1 for item in distances if item[0] < 64.0),
|
||||
"pairs_below_2000_m": sum(1 for item in distances if item[0] < 2000.0),
|
||||
"closest_pairs": [
|
||||
{
|
||||
"distance_m": item[0],
|
||||
"left_sample": item[1],
|
||||
"left_split": item[2],
|
||||
"right_sample": item[3],
|
||||
"right_split": item[4],
|
||||
}
|
||||
for item in distances[:20]
|
||||
],
|
||||
"claim_boundary": (
|
||||
"AOI bounding-box distance is a screening check only; it does not prove "
|
||||
"municipality, flight-strip, building-instance or imagery independence."
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def _dhash(path: Path) -> int:
|
||||
from PIL import Image
|
||||
|
||||
with Image.open(path) as image:
|
||||
resized = image.convert("L").resize((9, 8))
|
||||
pixel_source = (
|
||||
resized.get_flattened_data()
|
||||
if hasattr(resized, "get_flattened_data")
|
||||
else resized.getdata()
|
||||
)
|
||||
pixels = list(pixel_source)
|
||||
value = 0
|
||||
for row in range(8):
|
||||
offset = row * 9
|
||||
for column in range(8):
|
||||
value = (value << 1) | int(pixels[offset + column] > pixels[offset + column + 1])
|
||||
return value
|
||||
|
||||
|
||||
def cross_split_dhash_summary(samples: list[dict[str, Any]]) -> dict[str, Any]:
|
||||
hashes: list[tuple[str, str, int]] = []
|
||||
failures: list[dict[str, str]] = []
|
||||
for sample in samples:
|
||||
raster_path = Path(str(sample.get("raster_path") or ""))
|
||||
if not raster_path.is_file():
|
||||
failures.append({"sample_slug": str(sample.get("sample_slug")), "reason": "missing_raster"})
|
||||
continue
|
||||
try:
|
||||
hashes.append(
|
||||
(
|
||||
str(sample.get("sample_slug")),
|
||||
str(sample.get("split")),
|
||||
_dhash(raster_path),
|
||||
)
|
||||
)
|
||||
except Exception as exc:
|
||||
failures.append({"sample_slug": str(sample.get("sample_slug")), "reason": str(exc)})
|
||||
|
||||
matches: list[dict[str, Any]] = []
|
||||
minimum: int | None = None
|
||||
compared = 0
|
||||
for left, right in combinations(hashes, 2):
|
||||
if left[1] == right[1]:
|
||||
continue
|
||||
compared += 1
|
||||
distance = (left[2] ^ right[2]).bit_count()
|
||||
minimum = distance if minimum is None else min(minimum, distance)
|
||||
if distance <= 4:
|
||||
matches.append(
|
||||
{
|
||||
"left_sample": left[0],
|
||||
"left_split": left[1],
|
||||
"right_sample": right[0],
|
||||
"right_split": right[1],
|
||||
"hamming_distance": distance,
|
||||
}
|
||||
)
|
||||
return {
|
||||
"algorithm": "64-bit difference hash over 9x8 grayscale resize",
|
||||
"screened_raster_count": len(hashes),
|
||||
"cross_split_pair_count": compared,
|
||||
"minimum_hamming_distance": minimum,
|
||||
"pairs_at_or_below_4": matches,
|
||||
"failures": failures,
|
||||
"claim_boundary": (
|
||||
"This bounded perceptual screen is not semantic, instance-level or "
|
||||
"flight-strip deduplication and cannot establish split independence."
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def corpus_audit_summary(path: Path) -> dict[str, Any]:
|
||||
payload = json.loads(path.read_text(encoding="utf-8"))
|
||||
keys = (
|
||||
"status",
|
||||
"dataset_version",
|
||||
"manifest_immutable",
|
||||
"sample_count",
|
||||
"split_counts",
|
||||
"input_feature_count",
|
||||
"accepted_feature_count",
|
||||
"decision_counts",
|
||||
"temporal_unknown_sample_count",
|
||||
"spatial_leakage_status",
|
||||
"reviewed_sample_count",
|
||||
"review_complete",
|
||||
"failures",
|
||||
)
|
||||
return {
|
||||
**artifact_record(path),
|
||||
**{key: payload.get(key) for key in keys},
|
||||
"review_queue_count": len(payload.get("review_queue") or []),
|
||||
"claim_boundary": (
|
||||
"Automated status and an empty failure list do not substitute for "
|
||||
"human label acceptance."
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def calibration_summary(path: Path) -> dict[str, Any]:
|
||||
payload = json.loads(path.read_text(encoding="utf-8"))
|
||||
sweeps = payload.get("sweeps") if isinstance(payload.get("sweeps"), list) else []
|
||||
|
||||
def sweep_at(threshold: float) -> dict[str, Any] | None:
|
||||
return next(
|
||||
(
|
||||
{
|
||||
"threshold": item.get("threshold"),
|
||||
"aggregate": item.get("aggregate"),
|
||||
"regions": item.get("regions"),
|
||||
"pure_empty_false_positives": item.get("pure_empty_false_positives"),
|
||||
}
|
||||
for item in sweeps
|
||||
if abs(float(item.get("threshold")) - threshold) < 1e-12
|
||||
),
|
||||
None,
|
||||
)
|
||||
|
||||
return {
|
||||
**artifact_record(path),
|
||||
"model": payload.get("model"),
|
||||
"summary": payload.get("summary"),
|
||||
"split": payload.get("split"),
|
||||
"match_iou": payload.get("match_iou"),
|
||||
"tile_count": payload.get("tile_count"),
|
||||
"inference_imgsz": payload.get("inference_imgsz"),
|
||||
"sweep_count": len(sweeps),
|
||||
"threshold_0_15": sweep_at(0.15),
|
||||
"threshold_0_02": sweep_at(0.02),
|
||||
"claim_boundary": (
|
||||
"This is calibration-split, tile-level bbox evidence; it is not a "
|
||||
"protected-test, unique-building or national release result."
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def inventory(root: Path) -> dict[str, Any]:
|
||||
model_files = sorted(
|
||||
path
|
||||
for path in (root / "models").glob("*")
|
||||
if path.is_file() and path.suffix.lower() in {".pt", ".pth", ".onnx", ".engine", ".ckpt", ".safetensors"}
|
||||
)
|
||||
training_root = root / "storage" / "training"
|
||||
checkpoints = sorted(
|
||||
(
|
||||
path
|
||||
for path in training_root.glob("building-be-*/**/*")
|
||||
if path.is_file() and path.suffix.lower() in {".pt", ".pth", ".ckpt", ".onnx", ".engine", ".safetensors"}
|
||||
),
|
||||
key=lambda path: path.stat().st_mtime,
|
||||
reverse=True,
|
||||
)
|
||||
reports = sorted(
|
||||
(
|
||||
path
|
||||
for path in training_root.glob("building-be-*/**/*.json")
|
||||
if path.is_file()
|
||||
),
|
||||
key=lambda path: path.stat().st_mtime,
|
||||
reverse=True,
|
||||
)
|
||||
manifests = sorted(
|
||||
(root / "storage" / "operator-data").glob("building-be-*/operator_samples_manifest.json"),
|
||||
key=lambda path: path.stat().st_mtime,
|
||||
reverse=True,
|
||||
)
|
||||
return {
|
||||
"model_assets": [artifact_record(path) for path in model_files],
|
||||
"training_checkpoint_count": len(checkpoints),
|
||||
"training_checkpoint_bytes": sum(path.stat().st_size for path in checkpoints),
|
||||
"newest_training_checkpoints": [
|
||||
artifact_record(path, include_hash=False) for path in checkpoints[:30]
|
||||
],
|
||||
"training_json_report_count": len(reports),
|
||||
"newest_training_json_reports": [
|
||||
artifact_record(path, include_hash=False) for path in reports[:100]
|
||||
],
|
||||
"operator_manifest_count": len(manifests),
|
||||
"operator_manifests": [artifact_record(path) for path in manifests],
|
||||
}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Collect a bounded, read-only ML/data-lineage snapshot from the deployed GeoIntel volume."
|
||||
)
|
||||
parser.add_argument("--app-root", default="/app")
|
||||
args = parser.parse_args()
|
||||
root = Path(args.app_root).resolve()
|
||||
|
||||
paths = {
|
||||
"v56_audit": root / "storage/training/building-be-v56-corpus-audit-r1/belgium-building-corpus-audit.json",
|
||||
"v56_manifest": root / "storage/operator-data/building-be-v56-hard-negative-instance-roofs-r1/operator_samples_manifest.json",
|
||||
"v58_calibration": root / "storage/training/building-be-v58-v56-clean-pretrained-r1/preview-epoch-015/calibration.json",
|
||||
"v62_calibration": root / "storage/training/building-be-v62-v58-flanders-balanced-transfer-r1/preview-epoch-006/calibration-routed.json",
|
||||
"v66_manifest": root / "storage/operator-data/building-be-v66-lowrise-temporal-r1/operator_samples_manifest.json",
|
||||
}
|
||||
missing = [str(path) for path in paths.values() if not path.is_file()]
|
||||
if missing:
|
||||
raise SystemExit("Required lineage artifacts are missing: " + ", ".join(missing))
|
||||
|
||||
payload = {
|
||||
"schema_version": 1,
|
||||
"captured_at": datetime.now(timezone.utc).isoformat(),
|
||||
"read_only": True,
|
||||
"inventory": inventory(root),
|
||||
"v56": {
|
||||
"corpus_audit": corpus_audit_summary(paths["v56_audit"]),
|
||||
"manifest": manifest_summary(paths["v56_manifest"]),
|
||||
},
|
||||
"v58": {"calibration": calibration_summary(paths["v58_calibration"])},
|
||||
"v62": {"calibration": calibration_summary(paths["v62_calibration"])},
|
||||
"v66": {"manifest": manifest_summary(paths["v66_manifest"])},
|
||||
"global_claim_boundary": (
|
||||
"Inventory and historical calibration evidence do not prove human "
|
||||
"label acceptance, strict split independence, calibration, protected-test "
|
||||
"performance, national validity or release readiness."
|
||||
),
|
||||
}
|
||||
print(json.dumps(payload, indent=2, sort_keys=True))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,334 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Emit a bounded read-only database/runtime integrity snapshot as JSON.
|
||||
|
||||
Run inside the GeoIntel application container. Every SQL statement has a
|
||||
timeout; the script never writes application rows or storage artifacts.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import importlib.metadata
|
||||
import json
|
||||
import os
|
||||
import platform
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import text
|
||||
|
||||
from app.db.session import engine
|
||||
|
||||
|
||||
TABLES = (
|
||||
"projects",
|
||||
"areas",
|
||||
"datasets",
|
||||
"dataset_versions",
|
||||
"vector_features",
|
||||
"jobs",
|
||||
"analysis_runs",
|
||||
"detections",
|
||||
"segmentations",
|
||||
"quality_checks",
|
||||
"metrics",
|
||||
"exports",
|
||||
"detection_reviews",
|
||||
"aoi_operations",
|
||||
"aoi_operation_partitions",
|
||||
)
|
||||
PATH_QUERIES = {
|
||||
"datasets": "SELECT id::text, storage_path FROM datasets WHERE storage_path IS NOT NULL",
|
||||
"dataset_versions": "SELECT id::text, storage_path FROM dataset_versions WHERE storage_path IS NOT NULL",
|
||||
"exports": "SELECT id::text, storage_path FROM exports WHERE storage_path IS NOT NULL",
|
||||
}
|
||||
|
||||
|
||||
def now() -> str:
|
||||
return datetime.now(timezone.utc).isoformat()
|
||||
|
||||
|
||||
def sha256_file(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as handle:
|
||||
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def package_version(name: str) -> str | None:
|
||||
try:
|
||||
return importlib.metadata.version(name)
|
||||
except importlib.metadata.PackageNotFoundError:
|
||||
return None
|
||||
|
||||
|
||||
def normalize_path(raw: str, storage_root: Path) -> Path:
|
||||
candidate = Path(raw)
|
||||
if candidate.is_absolute():
|
||||
return candidate
|
||||
normalized = raw.replace("\\", "/")
|
||||
if normalized.startswith("storage/"):
|
||||
normalized = normalized.removeprefix("storage/")
|
||||
return storage_root / normalized
|
||||
|
||||
|
||||
def rows(connection: Any, statement: str, parameters: dict[str, Any] | None = None) -> list[dict[str, Any]]:
|
||||
result = connection.execute(text(statement), parameters or {})
|
||||
return [dict(row._mapping) for row in result]
|
||||
|
||||
|
||||
def scalar(connection: Any, statement: str) -> int:
|
||||
return int(connection.execute(text(statement)).scalar() or 0)
|
||||
|
||||
|
||||
def safe_query(connection: Any, name: str, statement: str) -> dict[str, Any]:
|
||||
try:
|
||||
return {"status": "ok", "rows": rows(connection, statement)}
|
||||
except Exception as exc:
|
||||
return {
|
||||
"status": "error",
|
||||
"error_type": type(exc).__name__,
|
||||
"message": str(exc).splitlines()[0][:500],
|
||||
"query_name": name,
|
||||
}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
storage_root = Path(os.environ.get("GEOINTEL_STORAGE_ROOT", "/app/storage")).resolve()
|
||||
report: dict[str, Any] = {
|
||||
"schema_version": 1,
|
||||
"generated_at": now(),
|
||||
"mode": "read_only",
|
||||
"statement_timeout_ms": 30000,
|
||||
"runtime": {
|
||||
"python": platform.python_version(),
|
||||
"platform": platform.platform(),
|
||||
"packages": {
|
||||
name: package_version(name)
|
||||
for name in (
|
||||
"geointel-backend",
|
||||
"fastapi",
|
||||
"sqlalchemy",
|
||||
"geoalchemy2",
|
||||
"shapely",
|
||||
"pyproj",
|
||||
"rasterio",
|
||||
"geopandas",
|
||||
"torch",
|
||||
"torchvision",
|
||||
"ultralytics",
|
||||
)
|
||||
},
|
||||
},
|
||||
"storage_root": str(storage_root),
|
||||
}
|
||||
try:
|
||||
import torch
|
||||
|
||||
report["runtime"]["cuda"] = {
|
||||
"available": torch.cuda.is_available(),
|
||||
"runtime_version": torch.version.cuda,
|
||||
"device_count": torch.cuda.device_count(),
|
||||
"device_names": [
|
||||
torch.cuda.get_device_name(index) for index in range(torch.cuda.device_count())
|
||||
],
|
||||
}
|
||||
except Exception as exc:
|
||||
report["runtime"]["cuda"] = {"available": False, "error": type(exc).__name__}
|
||||
|
||||
model_path = Path(os.environ.get("YOLO_MODEL_PATH", ""))
|
||||
report["active_model"] = {
|
||||
"configured_path": str(model_path) if str(model_path) else None,
|
||||
"exists": model_path.is_file(),
|
||||
"size_bytes": model_path.stat().st_size if model_path.is_file() else None,
|
||||
"sha256": sha256_file(model_path) if model_path.is_file() else None,
|
||||
"model_id": os.environ.get("YOLO_MODEL_ID"),
|
||||
"model_version": os.environ.get("YOLO_MODEL_VERSION"),
|
||||
"classes": os.environ.get("YOLO_MODEL_CLASSES"),
|
||||
"device": os.environ.get("YOLO_DEVICE"),
|
||||
"require_cuda": os.environ.get("YOLO_REQUIRE_CUDA"),
|
||||
"validated_area_names": os.environ.get("YOLO_VALIDATED_AREA_NAMES"),
|
||||
"validation_scope_enforced": os.environ.get("YOLO_ENFORCE_VALIDATION_SCOPE"),
|
||||
}
|
||||
|
||||
with engine.connect().execution_options(isolation_level="AUTOCOMMIT") as connection:
|
||||
connection.execute(text("SET statement_timeout TO '30s'"))
|
||||
report["database"] = {
|
||||
"version": connection.execute(text("SELECT version()")).scalar(),
|
||||
"postgis_version": connection.execute(text("SELECT PostGIS_Full_Version()")).scalar(),
|
||||
"migration_heads": [row["version_num"] for row in rows(connection, "SELECT version_num FROM alembic_version")],
|
||||
"table_counts": {table: scalar(connection, f'SELECT count(*) FROM "{table}"') for table in TABLES},
|
||||
}
|
||||
report["database"]["dataset_statuses"] = rows(
|
||||
connection,
|
||||
"SELECT status, count(*)::bigint AS count FROM datasets GROUP BY status ORDER BY status",
|
||||
)
|
||||
report["database"]["job_statuses"] = rows(
|
||||
connection,
|
||||
"SELECT status, count(*)::bigint AS count FROM jobs GROUP BY status ORDER BY status",
|
||||
)
|
||||
report["database"]["analysis_statuses"] = rows(
|
||||
connection,
|
||||
"SELECT status, count(*)::bigint AS count FROM analysis_runs GROUP BY status ORDER BY status",
|
||||
)
|
||||
report["database"]["dataset_lineage_gaps"] = rows(
|
||||
connection,
|
||||
"""
|
||||
SELECT
|
||||
count(*)::bigint AS total,
|
||||
count(*) FILTER (WHERE crs IS NULL OR btrim(crs) = '')::bigint AS missing_crs,
|
||||
count(*) FILTER (WHERE checksum_sha256 IS NULL OR btrim(checksum_sha256) = '')::bigint AS missing_checksum,
|
||||
count(*) FILTER (WHERE source_version IS NULL OR btrim(source_version) = '')::bigint AS missing_source_version,
|
||||
count(*) FILTER (WHERE source_metadata IS NULL OR source_metadata::text = '{}')::bigint AS missing_source_metadata,
|
||||
count(*) FILTER (WHERE provenance_metadata IS NULL OR provenance_metadata::text = '{}')::bigint AS missing_provenance_metadata,
|
||||
count(*) FILTER (WHERE imported_at IS NULL)::bigint AS missing_imported_at,
|
||||
count(*) FILTER (WHERE observed_at IS NULL)::bigint AS missing_observed_at
|
||||
FROM datasets
|
||||
""",
|
||||
)
|
||||
report["database"]["dataset_version_gaps"] = rows(
|
||||
connection,
|
||||
"""
|
||||
SELECT
|
||||
count(*)::bigint AS total,
|
||||
count(*) FILTER (WHERE checksum_sha256 IS NULL OR btrim(checksum_sha256) = '')::bigint AS missing_checksum,
|
||||
count(*) FILTER (WHERE source_metadata IS NULL OR source_metadata::text = '{}')::bigint AS missing_source_metadata,
|
||||
count(*) FILTER (WHERE provenance_metadata IS NULL OR provenance_metadata::text = '{}')::bigint AS missing_provenance_metadata,
|
||||
count(*) FILTER (WHERE storage_path IS NULL OR btrim(storage_path) = '')::bigint AS missing_storage_path
|
||||
FROM dataset_versions
|
||||
""",
|
||||
)
|
||||
report["database"]["geometry_integrity"] = {}
|
||||
for table, nullable in (
|
||||
("areas", False),
|
||||
("vector_features", False),
|
||||
("detections", True),
|
||||
("segmentations", False),
|
||||
):
|
||||
where = "WHERE geometry IS NOT NULL" if nullable else ""
|
||||
statement = f"""
|
||||
SELECT
|
||||
count(*)::bigint AS populated,
|
||||
count(*) FILTER (WHERE ST_IsEmpty(geometry))::bigint AS empty,
|
||||
count(*) FILTER (WHERE NOT ST_IsValid(geometry))::bigint AS invalid,
|
||||
count(*) FILTER (WHERE ST_SRID(geometry) <> 4326)::bigint AS wrong_srid,
|
||||
count(*) FILTER (
|
||||
WHERE ST_XMin(Box3D(geometry)) < -180
|
||||
OR ST_XMax(Box3D(geometry)) > 180
|
||||
OR ST_YMin(Box3D(geometry)) < -90
|
||||
OR ST_YMax(Box3D(geometry)) > 90
|
||||
)::bigint AS outside_epsg4326_domain
|
||||
FROM {table} {where}
|
||||
"""
|
||||
report["database"]["geometry_integrity"][table] = safe_query(
|
||||
connection, f"{table}_geometry_integrity", statement
|
||||
)
|
||||
report["database"]["outside_domain_detection_records"] = rows(
|
||||
connection,
|
||||
"""
|
||||
SELECT
|
||||
d.id::text AS detection_id,
|
||||
d.analysis_run_id::text AS analysis_run_id,
|
||||
d.dataset_id::text AS dataset_id,
|
||||
p.name AS project_name,
|
||||
ds.name AS dataset_name,
|
||||
ds.source_name AS dataset_source_name,
|
||||
ar.analysis_type,
|
||||
ar.status AS analysis_status,
|
||||
d.model_name,
|
||||
d.model_version,
|
||||
d.class_name,
|
||||
d.confidence,
|
||||
d.source_tile_path,
|
||||
d.bbox_json,
|
||||
ST_XMin(Box3D(d.geometry)) AS min_x,
|
||||
ST_YMin(Box3D(d.geometry)) AS min_y,
|
||||
ST_XMax(Box3D(d.geometry)) AS max_x,
|
||||
ST_YMax(Box3D(d.geometry)) AS max_y,
|
||||
d.created_at
|
||||
FROM detections d
|
||||
LEFT JOIN analysis_runs ar ON ar.id = d.analysis_run_id
|
||||
LEFT JOIN datasets ds ON ds.id = d.dataset_id
|
||||
LEFT JOIN projects p ON p.id = d.project_id
|
||||
WHERE d.geometry IS NOT NULL
|
||||
AND (
|
||||
ST_XMin(Box3D(d.geometry)) < -180
|
||||
OR ST_XMax(Box3D(d.geometry)) > 180
|
||||
OR ST_YMin(Box3D(d.geometry)) < -90
|
||||
OR ST_YMax(Box3D(d.geometry)) > 90
|
||||
)
|
||||
ORDER BY d.created_at, d.id
|
||||
LIMIT 100
|
||||
""",
|
||||
)
|
||||
report["database"]["confidence_integrity"] = rows(
|
||||
connection,
|
||||
"""
|
||||
SELECT
|
||||
(SELECT count(*) FROM detections WHERE confidence < 0 OR confidence > 1)::bigint
|
||||
AS detections_outside_unit_interval,
|
||||
(SELECT count(*) FROM segmentations
|
||||
WHERE confidence IS NOT NULL AND (confidence < 0 OR confidence > 1))::bigint
|
||||
AS segmentations_outside_unit_interval
|
||||
""",
|
||||
)
|
||||
report["database"]["metric_nulls"] = rows(
|
||||
connection,
|
||||
"""
|
||||
SELECT metric_key, count(*)::bigint AS total,
|
||||
count(*) FILTER (WHERE metric_value IS NULL)::bigint AS null_values
|
||||
FROM metrics GROUP BY metric_key ORDER BY metric_key
|
||||
""",
|
||||
)
|
||||
report["database"]["model_run_summary"] = rows(
|
||||
connection,
|
||||
"""
|
||||
SELECT analysis_type, status, coalesce(model_name, '<none>') AS model_name,
|
||||
coalesce(model_version, '<none>') AS model_version, count(*)::bigint AS count
|
||||
FROM analysis_runs
|
||||
GROUP BY analysis_type, status, model_name, model_version
|
||||
ORDER BY count(*) DESC, analysis_type
|
||||
LIMIT 100
|
||||
""",
|
||||
)
|
||||
report["database"]["source_summary"] = rows(
|
||||
connection,
|
||||
"""
|
||||
SELECT coalesce(source_name, source, '<none>') AS source_name,
|
||||
status, count(*)::bigint AS count,
|
||||
count(*) FILTER (WHERE dataset_role = 'reference')::bigint AS reference_count
|
||||
FROM datasets
|
||||
GROUP BY coalesce(source_name, source, '<none>'), status
|
||||
ORDER BY count(*) DESC, source_name
|
||||
LIMIT 200
|
||||
""",
|
||||
)
|
||||
|
||||
path_records = []
|
||||
for table, statement in PATH_QUERIES.items():
|
||||
for row in rows(connection, statement):
|
||||
path = normalize_path(row["storage_path"], storage_root)
|
||||
path_records.append({
|
||||
"table": table,
|
||||
"id": row["id"],
|
||||
"storage_path": row["storage_path"],
|
||||
"resolved_path": str(path),
|
||||
"exists": path.is_file() or path.is_dir(),
|
||||
})
|
||||
missing = [row for row in path_records if not row["exists"]]
|
||||
report["storage_references"] = {
|
||||
"checked_count": len(path_records),
|
||||
"missing_count": len(missing),
|
||||
"missing_records": missing[:500],
|
||||
"records_truncated": len(missing) > 500,
|
||||
"scope": "direct datasets, dataset_versions and exports storage_path columns",
|
||||
}
|
||||
|
||||
print(json.dumps(report, ensure_ascii=False, indent=2, sort_keys=True, default=str))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,271 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from uuid import UUID
|
||||
|
||||
from geoalchemy2.shape import to_shape
|
||||
from shapely.geometry import Point, box
|
||||
|
||||
import sys
|
||||
|
||||
|
||||
REPOSITORY_ROOT = Path(__file__).resolve().parents[1]
|
||||
BACKEND_ROOT = REPOSITORY_ROOT / "backend"
|
||||
if str(BACKEND_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(BACKEND_ROOT))
|
||||
|
||||
from app.core.config import Settings # noqa: E402
|
||||
from app.schemas.area import AreaUpdate # noqa: E402
|
||||
from app.services.coverage_registry_service import ( # noqa: E402
|
||||
CoverageRegistryService,
|
||||
SOURCE_DEFINITIONS,
|
||||
)
|
||||
from app.services.detection_service import DetectionService # noqa: E402
|
||||
from app.services.vector_feature_service import VectorFeatureService # noqa: E402
|
||||
|
||||
|
||||
FIXED_UUID = UUID("00000000-0000-4000-8000-000000000001")
|
||||
|
||||
|
||||
def _dataset(*, layer: str, bbox_values: list[float], source_name: str = "spw_picc") -> SimpleNamespace:
|
||||
return SimpleNamespace(
|
||||
id=FIXED_UUID,
|
||||
status="ready",
|
||||
source_name=source_name,
|
||||
reference_layer_name=layer,
|
||||
source_metadata={"coverage_zones": ["wallonia"], "bbox_epsg4326": bbox_values},
|
||||
provenance_metadata={},
|
||||
observed_at=None,
|
||||
source_version="forensic-reproduction",
|
||||
resolution_json=None,
|
||||
checksum_sha256="forensic-only",
|
||||
)
|
||||
|
||||
|
||||
def _coverage_cross_theme_contamination() -> dict:
|
||||
definition = next(
|
||||
item for item in SOURCE_DEFINITIONS if item.contract.source_name == "spw_geoportail"
|
||||
)
|
||||
selection = box(4.50, 50.50, 4.70, 50.60)
|
||||
building = _dataset(layer="buildings", bbox_values=[4.55, 50.52, 4.56, 50.53])
|
||||
road = _dataset(layer="roads", bbox_values=[4.50, 50.50, 4.70, 50.60])
|
||||
matches, fully_covered = CoverageRegistryService._matching_datasets(
|
||||
[building, road],
|
||||
definition,
|
||||
"buildings",
|
||||
"wallonia",
|
||||
selection,
|
||||
)
|
||||
observed_ids = [str(item.id) for item in matches]
|
||||
reproduced = observed_ids == [str(building.id)] and fully_covered is True
|
||||
return {
|
||||
"id": "P1-COV-001",
|
||||
"severity": "critical",
|
||||
"source": "backend/app/services/coverage_registry_service.py:481-505",
|
||||
"expected": "A small buildings partition remains partial; a roads bbox cannot complete buildings coverage.",
|
||||
"observed": {"matched_dataset_ids": observed_ids, "fully_covered": fully_covered},
|
||||
"reproduced": reproduced,
|
||||
}
|
||||
|
||||
|
||||
def _meter_buffer_as_degrees() -> dict:
|
||||
geometry = Point(5.0, 51.0)
|
||||
buffered = geometry.buffer(100.0)
|
||||
bounds = [float(value) for value in buffered.bounds]
|
||||
reproduced = round(bounds[2] - bounds[0], 6) == 200.0
|
||||
return {
|
||||
"id": "P1-CRS-001",
|
||||
"severity": "critical",
|
||||
"source": "backend/app/services/vector_operations_service.py:179-203",
|
||||
"expected": "A 100 metre buffer is projected to a metric CRS and spans roughly hundreds of metres.",
|
||||
"observed": {"bounds_epsg4326": bounds, "longitude_span_degrees": bounds[2] - bounds[0]},
|
||||
"reproduced": reproduced,
|
||||
}
|
||||
|
||||
|
||||
def _lambert_feature_mislabeled() -> dict:
|
||||
row = VectorFeatureService._feature_row(
|
||||
FIXED_UUID,
|
||||
{
|
||||
"type": "Feature",
|
||||
"geometry": {"type": "Point", "coordinates": [150000.0, 210000.0]},
|
||||
"properties": {},
|
||||
},
|
||||
0,
|
||||
None,
|
||||
)
|
||||
assert row is not None
|
||||
geometry = to_shape(row.geometry)
|
||||
reproduced = (
|
||||
int(row.geometry.srid) == 4326
|
||||
and float(geometry.x) == 150000.0
|
||||
and float(geometry.y) == 210000.0
|
||||
)
|
||||
return {
|
||||
"id": "P1-CRS-002",
|
||||
"severity": "critical",
|
||||
"source": "backend/app/services/vector_feature_service.py:271-299",
|
||||
"expected": "Non-WGS84 input is transformed to EPSG:4326 or rejected before persistence.",
|
||||
"observed": {
|
||||
"stored_srid": int(row.geometry.srid),
|
||||
"stored_coordinates": [float(geometry.x), float(geometry.y)],
|
||||
},
|
||||
"reproduced": reproduced,
|
||||
}
|
||||
|
||||
|
||||
def _authority_spoof() -> dict:
|
||||
selection = box(4.9, 50.9, 5.0, 51.0)
|
||||
dataset = SimpleNamespace(
|
||||
id=FIXED_UUID,
|
||||
status="ready",
|
||||
source_name="grb",
|
||||
reference_layer_name="buildings",
|
||||
source_metadata={
|
||||
"coverage_zones": ["flanders"],
|
||||
"bbox_epsg4326": [4.8, 50.8, 5.1, 51.1],
|
||||
},
|
||||
provenance_metadata={"provided_by": "caller"},
|
||||
observed_at=None,
|
||||
source_version="caller-provided",
|
||||
resolution_json=None,
|
||||
checksum_sha256="caller-provided",
|
||||
crs="EPSG:4326",
|
||||
)
|
||||
item = CoverageRegistryService._resolve_item(
|
||||
zone="flanders",
|
||||
theme="buildings",
|
||||
datasets=[dataset],
|
||||
selection=selection,
|
||||
)
|
||||
authority = item.evidence[0].authority_level if item.evidence else None
|
||||
reproduced = item.status == "operational" and authority == "authoritative"
|
||||
return {
|
||||
"id": "P1-AUTH-001",
|
||||
"severity": "critical",
|
||||
"source": (
|
||||
"backend/app/api/routes/datasets.py:142-163; "
|
||||
"backend/app/services/coverage_registry_service.py:463-559"
|
||||
),
|
||||
"expected": "Only server-attested source identities can produce authoritative operational coverage.",
|
||||
"observed": {
|
||||
"caller_controlled_source_name": dataset.source_name,
|
||||
"coverage_status": item.status,
|
||||
"reported_authority": authority,
|
||||
},
|
||||
"reproduced": reproduced,
|
||||
}
|
||||
|
||||
|
||||
def _mutable_name_model_scope() -> dict:
|
||||
area = SimpleNamespace(name="Mol validation bypass", geometry=box(-75.0, 35.0, -74.9, 35.1))
|
||||
dataset = SimpleNamespace(id=FIXED_UUID, area_id=FIXED_UUID)
|
||||
|
||||
class FakeSession:
|
||||
@staticmethod
|
||||
def get(_model, _identifier):
|
||||
return area
|
||||
|
||||
settings = Settings().model_copy(
|
||||
update={"yolo_validated_area_names": "Mol,Kempen"}
|
||||
)
|
||||
accepted = True
|
||||
try:
|
||||
DetectionService._validate_model_area_scope(FakeSession(), dataset, settings)
|
||||
except Exception:
|
||||
accepted = False
|
||||
return {
|
||||
"id": "P1-AI-001",
|
||||
"severity": "critical",
|
||||
"source": "backend/app/services/detection_service.py:223-232",
|
||||
"expected": "Validation scope is bound to immutable geometry/source/checksum evidence.",
|
||||
"observed": {
|
||||
"area_name": area.name,
|
||||
"geometry_bounds": [float(value) for value in area.geometry.bounds],
|
||||
"accepted": accepted,
|
||||
},
|
||||
"reproduced": accepted,
|
||||
}
|
||||
|
||||
|
||||
def _mutable_name_legal_scope() -> dict:
|
||||
selection = box(4.9, 50.9, 5.0, 51.0)
|
||||
flemish_geometry = box(2.5, 50.7, 5.9, 51.5)
|
||||
canonical = [SimpleNamespace(name="Flanders", geometry=flemish_geometry)]
|
||||
renamed = [SimpleNamespace(name="Vlaanderen", geometry=flemish_geometry)]
|
||||
canonical_zones, canonical_outside = CoverageRegistryService._intersected_zones(
|
||||
canonical, selection
|
||||
)
|
||||
renamed_zones, renamed_outside = CoverageRegistryService._intersected_zones(
|
||||
renamed, selection
|
||||
)
|
||||
reproduced = (
|
||||
canonical_zones == ["flanders"]
|
||||
and canonical_outside is False
|
||||
and renamed_zones == []
|
||||
and renamed_outside is True
|
||||
)
|
||||
return {
|
||||
"id": "P1-COV-002",
|
||||
"severity": "high",
|
||||
"source": "backend/app/services/coverage_registry_service.py:56-65,425-447",
|
||||
"expected": "Renaming an Area cannot change its legal coverage-zone identity.",
|
||||
"observed": {
|
||||
"canonical": {"zones": canonical_zones, "outside": canonical_outside},
|
||||
"renamed_same_geometry": {"zones": renamed_zones, "outside": renamed_outside},
|
||||
},
|
||||
"reproduced": reproduced,
|
||||
}
|
||||
|
||||
|
||||
def _area_patch_ignores_geometry() -> dict:
|
||||
payload = AreaUpdate.model_validate(
|
||||
{
|
||||
"name": "Renamed",
|
||||
"geometry": {
|
||||
"type": "Polygon",
|
||||
"coordinates": [[[4.0, 50.0], [5.0, 50.0], [5.0, 51.0], [4.0, 50.0]]],
|
||||
},
|
||||
}
|
||||
)
|
||||
parsed = payload.model_dump()
|
||||
reproduced = "geometry" not in parsed
|
||||
return {
|
||||
"id": "P1-API-001",
|
||||
"severity": "high",
|
||||
"source": "backend/app/schemas/area.py:15-17; backend/app/services/area_service.py:154-171",
|
||||
"expected": "PATCH /areas/{area_id} either validates and applies geometry or rejects the field.",
|
||||
"observed": {"parsed_payload": parsed, "geometry_silently_ignored": reproduced},
|
||||
"reproduced": reproduced,
|
||||
}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
findings = [
|
||||
_coverage_cross_theme_contamination(),
|
||||
_meter_buffer_as_degrees(),
|
||||
_lambert_feature_mislabeled(),
|
||||
_authority_spoof(),
|
||||
_mutable_name_model_scope(),
|
||||
_mutable_name_legal_scope(),
|
||||
_area_patch_ignores_geometry(),
|
||||
]
|
||||
reproduced_count = sum(bool(item["reproduced"]) for item in findings)
|
||||
payload = {
|
||||
"schema_version": 1,
|
||||
"purpose": "Read-only deterministic reproductions of Phase-1 contract violations.",
|
||||
"findings": findings,
|
||||
"summary": {
|
||||
"total": len(findings),
|
||||
"reproduced": reproduced_count,
|
||||
"all_reproduced": reproduced_count == len(findings),
|
||||
},
|
||||
}
|
||||
print(json.dumps(payload, indent=2, sort_keys=True))
|
||||
return 0 if reproduced_count == len(findings) else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,365 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Create a bounded, read-only GeoIntel Phase-1 accuracy baseline."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
import subprocess
|
||||
from collections import Counter, defaultdict
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any, Iterable
|
||||
|
||||
SCHEMA_VERSION = 1
|
||||
SKIP_DIRS = {".git", ".pytest_cache", ".ruff_cache", ".venv", "__pycache__", "dist", "node_modules"}
|
||||
CODE_ROOTS = {
|
||||
"backend": "backend/app",
|
||||
"backend_tests": "backend/tests",
|
||||
"frontend": "frontend/src",
|
||||
"frontend_e2e": "frontend/e2e",
|
||||
"root_tests": "tests",
|
||||
"scripts": "scripts",
|
||||
"migrations": "backend/alembic/versions",
|
||||
}
|
||||
CODE_SUFFIXES = {".py", ".ts", ".tsx", ".js", ".mjs", ".sh", ".ps1"}
|
||||
ARTIFACT_ROOTS = ("models", "datasets", "data", "storage", "artifacts", "output")
|
||||
MODEL_SUFFIXES = {".pt", ".pth", ".onnx", ".engine", ".safetensors"}
|
||||
RASTER_SUFFIXES = {".tif", ".tiff", ".vrt", ".jp2"}
|
||||
VECTOR_SUFFIXES = {".geojson", ".gpkg", ".shp", ".fgb"}
|
||||
HASH_SUFFIXES = MODEL_SUFFIXES | RASTER_SUFFIXES | VECTOR_SUFFIXES | {
|
||||
".json", ".yaml", ".yml", ".csv", ".txt", ".md", ".lock"
|
||||
}
|
||||
MARKERS = {
|
||||
"fixture": re.compile(r"\bfixture\b", re.IGNORECASE),
|
||||
"mock": re.compile(r"\bmock(?:ed|ing|s)?\b", re.IGNORECASE),
|
||||
"placeholder": re.compile(r"\bplaceholder\b", re.IGNORECASE),
|
||||
"heuristic": re.compile(r"\bheuristic(?:s)?\b", re.IGNORECASE),
|
||||
"not_configured": re.compile(r"\bnot_configured\b", re.IGNORECASE),
|
||||
"todo": re.compile(r"\bTODO\b"),
|
||||
"fallback": re.compile(r"\b(?:fallback|fall back)\b", re.IGNORECASE),
|
||||
}
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--repo-root", type=Path, default=Path(__file__).resolve().parents[1])
|
||||
parser.add_argument("--output-dir", type=Path, required=True)
|
||||
parser.add_argument("--max-hash-bytes", type=int, default=64 * 1024 * 1024)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def now() -> str:
|
||||
return datetime.now(timezone.utc).isoformat()
|
||||
|
||||
|
||||
def sha256_file(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as handle:
|
||||
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def write_json(path: Path, payload: Any) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(
|
||||
json.dumps(payload, ensure_ascii=False, indent=2, sort_keys=True) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
|
||||
def git(repo: Path, *arguments: str) -> str:
|
||||
result = subprocess.run(
|
||||
["git", *arguments],
|
||||
cwd=repo,
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
encoding="utf-8",
|
||||
errors="replace",
|
||||
)
|
||||
return result.stdout
|
||||
|
||||
|
||||
def iter_files(root: Path) -> Iterable[Path]:
|
||||
if not root.is_dir():
|
||||
return
|
||||
for path in sorted(root.rglob("*")):
|
||||
if path.is_file() and not any(part in SKIP_DIRS for part in path.parts):
|
||||
yield path
|
||||
|
||||
|
||||
def line_count(path: Path) -> int:
|
||||
with path.open("r", encoding="utf-8", errors="replace") as handle:
|
||||
return sum(1 for _ in handle)
|
||||
|
||||
|
||||
def code_inventory(repo: Path) -> dict[str, Any]:
|
||||
groups: dict[str, dict[str, int]] = {}
|
||||
for name, relative in CODE_ROOTS.items():
|
||||
files = [path for path in iter_files(repo / relative) if path.suffix.lower() in CODE_SUFFIXES]
|
||||
groups[name] = {
|
||||
"file_count": len(files),
|
||||
"line_count": sum(line_count(path) for path in files),
|
||||
}
|
||||
|
||||
pytest_pattern = re.compile(r"^\s*(?:async\s+)?def\s+test_", re.MULTILINE)
|
||||
route_pattern = re.compile(r"@router\.(?:get|post|put|patch|delete)\s*\(")
|
||||
pytest_count = 0
|
||||
route_count = 0
|
||||
frontend_test_count = 0
|
||||
for base in (repo / "backend/tests", repo / "tests"):
|
||||
for path in iter_files(base):
|
||||
if path.suffix == ".py":
|
||||
pytest_count += len(pytest_pattern.findall(path.read_text(encoding="utf-8", errors="replace")))
|
||||
for path in iter_files(repo / "backend/app/api/routes"):
|
||||
if path.suffix == ".py":
|
||||
route_count += len(route_pattern.findall(path.read_text(encoding="utf-8", errors="replace")))
|
||||
for path in iter_files(repo / "frontend"):
|
||||
if ".test." in path.name.lower() or ".spec." in path.name.lower():
|
||||
frontend_test_count += 1
|
||||
return {
|
||||
"groups": groups,
|
||||
"pytest_test_function_count": pytest_count,
|
||||
"frontend_test_file_count": frontend_test_count,
|
||||
"api_route_decorator_count": route_count,
|
||||
}
|
||||
|
||||
|
||||
def migration_inventory(repo: Path) -> dict[str, Any]:
|
||||
revision_re = re.compile(r'^revision\s*(?::[^=]+)?=\s*["\x27]([^"\x27]+)["\x27]', re.MULTILINE)
|
||||
down_re = re.compile(
|
||||
r'^down_revision\s*(?::[^=]+)?=\s*(?:["\x27]([^"\x27]+)["\x27]|None)',
|
||||
re.MULTILINE,
|
||||
)
|
||||
rows = []
|
||||
for path in iter_files(repo / "backend/alembic/versions"):
|
||||
if path.suffix != ".py":
|
||||
continue
|
||||
text = path.read_text(encoding="utf-8", errors="replace")
|
||||
revision = revision_re.search(text)
|
||||
down = down_re.search(text)
|
||||
rows.append({
|
||||
"path": path.relative_to(repo).as_posix(),
|
||||
"revision": revision.group(1) if revision else None,
|
||||
"down_revision": down.group(1) if down and down.group(1) else None,
|
||||
})
|
||||
revisions = {row["revision"] for row in rows if row["revision"]}
|
||||
parents = {row["down_revision"] for row in rows if row["down_revision"]}
|
||||
return {
|
||||
"count": len(rows),
|
||||
"records": rows,
|
||||
"heads_from_static_chain": sorted(revisions - parents),
|
||||
"missing_revision_identifiers": sum(row["revision"] is None for row in rows),
|
||||
}
|
||||
|
||||
|
||||
def mirror_inventory(repo: Path, tracked: set[str]) -> dict[str, Any]:
|
||||
identical = 0
|
||||
different = []
|
||||
for relative in sorted(item for item in tracked if not item.startswith("geointel/")):
|
||||
mirror_relative = f"geointel/{relative}"
|
||||
if mirror_relative not in tracked:
|
||||
continue
|
||||
source = repo / relative
|
||||
mirror = repo / mirror_relative
|
||||
if not source.is_file() or not mirror.is_file():
|
||||
continue
|
||||
source_hash = sha256_file(source)
|
||||
mirror_hash = sha256_file(mirror)
|
||||
if source_hash == mirror_hash:
|
||||
identical += 1
|
||||
else:
|
||||
different.append({
|
||||
"path": relative,
|
||||
"root_sha256": source_hash,
|
||||
"mirror_sha256": mirror_hash,
|
||||
"root_size_bytes": source.stat().st_size,
|
||||
"mirror_size_bytes": mirror.stat().st_size,
|
||||
})
|
||||
return {
|
||||
"tracked_mirror_file_count": sum(item.startswith("geointel/") for item in tracked),
|
||||
"paired_identical_file_count": identical,
|
||||
"paired_different_file_count": len(different),
|
||||
"different_files": different,
|
||||
"risk": (
|
||||
"The tracked geointel/ repository mirror can create ambiguous imports, stale tests "
|
||||
"and local/deployment drift; Docker excludes it but local tools may not."
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def marker_inventory(repo: Path) -> dict[str, Any]:
|
||||
counts: Counter[str] = Counter()
|
||||
examples: dict[str, list[dict[str, Any]]] = defaultdict(list)
|
||||
for base in (repo / "backend/app", repo / "frontend/src", repo / "scripts"):
|
||||
for path in iter_files(base):
|
||||
if path.suffix.lower() not in CODE_SUFFIXES:
|
||||
continue
|
||||
relative = path.relative_to(repo).as_posix()
|
||||
for number, text in enumerate(
|
||||
path.read_text(encoding="utf-8", errors="replace").splitlines(), start=1
|
||||
):
|
||||
for name, pattern in MARKERS.items():
|
||||
if pattern.search(text):
|
||||
counts[name] += 1
|
||||
if len(examples[name]) < 25:
|
||||
examples[name].append({
|
||||
"path": relative,
|
||||
"line": number,
|
||||
"text": text.strip()[:240],
|
||||
})
|
||||
return {
|
||||
"counts": dict(sorted(counts.items())),
|
||||
"examples": dict(sorted(examples.items())),
|
||||
"interpretation": "Triage signals only; production impact requires a traced contract/runtime path.",
|
||||
}
|
||||
|
||||
|
||||
def artifact_role(path: Path) -> str:
|
||||
suffix = path.suffix.lower()
|
||||
name = path.name.lower()
|
||||
if suffix in MODEL_SUFFIXES:
|
||||
return "model_checkpoint"
|
||||
if "manifest" in name or name in {"dataset.yaml", "data.yaml"}:
|
||||
return "manifest"
|
||||
if any(token in name for token in ("audit", "evaluation", "assessment", "metric", "report")):
|
||||
return "evaluation_or_audit"
|
||||
if any(token in name for token in ("contact_sheet", "review")) or suffix in {".png", ".jpg", ".jpeg"}:
|
||||
return "visual_review"
|
||||
if suffix in RASTER_SUFFIXES:
|
||||
return "raster"
|
||||
if suffix in VECTOR_SUFFIXES:
|
||||
return "vector"
|
||||
if suffix in {".db", ".sqlite", ".sqlite3", ".wal", ".shm"} or ".db-" in name:
|
||||
return "database_runtime_state"
|
||||
if suffix == ".txt" and "label" in path.as_posix().lower():
|
||||
return "label"
|
||||
return "other"
|
||||
|
||||
|
||||
def artifact_inventory(
|
||||
repo: Path,
|
||||
tracked: set[str],
|
||||
output: Path,
|
||||
max_hash_bytes: int,
|
||||
) -> dict[str, Any]:
|
||||
records = []
|
||||
output = output.resolve()
|
||||
for root_name in ARTIFACT_ROOTS:
|
||||
for path in iter_files(repo / root_name):
|
||||
resolved = path.resolve()
|
||||
if resolved == output or output in resolved.parents:
|
||||
continue
|
||||
stat = path.stat()
|
||||
relative = path.relative_to(repo).as_posix()
|
||||
can_hash = stat.st_size <= max_hash_bytes and path.suffix.lower() in HASH_SUFFIXES
|
||||
records.append({
|
||||
"path": relative,
|
||||
"role": artifact_role(path),
|
||||
"size_bytes": stat.st_size,
|
||||
"modified_at": datetime.fromtimestamp(stat.st_mtime, timezone.utc).isoformat(),
|
||||
"tracked": relative in tracked,
|
||||
"sha256": sha256_file(path) if can_hash else None,
|
||||
"hash_omission_reason": None if can_hash else "suffix_or_size_limit",
|
||||
})
|
||||
roles = Counter(row["role"] for row in records)
|
||||
roots = Counter(row["path"].split("/", 1)[0] for row in records)
|
||||
return {
|
||||
"roots": list(ARTIFACT_ROOTS),
|
||||
"file_count": len(records),
|
||||
"total_size_bytes": sum(row["size_bytes"] for row in records),
|
||||
"role_counts": dict(sorted(roles.items())),
|
||||
"root_counts": dict(sorted(roots.items())),
|
||||
"model_checkpoint_count": roles.get("model_checkpoint", 0),
|
||||
"records": records,
|
||||
"limitations": [
|
||||
"Ignored Tower corpora and mounted model volumes can be absent locally.",
|
||||
"Large/non-evidence files are inventoried without a SHA-256 above the configured ceiling.",
|
||||
"Near-duplicate imagery and semantic label quality need dedicated corpus checks.",
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
args = parse_args()
|
||||
repo = args.repo_root.expanduser().resolve()
|
||||
output = args.output_dir.expanduser()
|
||||
output = output.resolve() if output.is_absolute() else (repo / output).resolve()
|
||||
if not (repo / ".git").exists():
|
||||
raise SystemExit(f"Not a Git repository root: {repo}")
|
||||
if output == repo:
|
||||
raise SystemExit("Output directory must not equal the repository root")
|
||||
|
||||
tracked = set(filter(None, git(repo, "ls-files").splitlines()))
|
||||
repository = {
|
||||
"schema_version": SCHEMA_VERSION,
|
||||
"generated_at": now(),
|
||||
"repo_root": str(repo),
|
||||
"git": {
|
||||
"branch": git(repo, "branch", "--show-current").strip(),
|
||||
"head": git(repo, "rev-parse", "HEAD").strip(),
|
||||
"status_porcelain": [line for line in git(repo, "status", "--short").splitlines() if line],
|
||||
"tracked_file_count": len(tracked),
|
||||
"top_level_tracked_counts": dict(sorted(Counter(
|
||||
item.split("/", 1)[0] for item in tracked
|
||||
).items())),
|
||||
},
|
||||
"code": code_inventory(repo),
|
||||
"migrations": migration_inventory(repo),
|
||||
"tracked_mirror": mirror_inventory(repo, tracked),
|
||||
}
|
||||
signals = marker_inventory(repo)
|
||||
artifacts = artifact_inventory(repo, tracked, output, args.max_hash_bytes)
|
||||
findings = []
|
||||
mirror = repository["tracked_mirror"]
|
||||
if mirror["tracked_mirror_file_count"]:
|
||||
findings.append({
|
||||
"id": "P1-REPO-001",
|
||||
"severity": "high",
|
||||
"title": "Tracked nested repository mirror creates ambiguous source state",
|
||||
"evidence": {
|
||||
"tracked_mirror_file_count": mirror["tracked_mirror_file_count"],
|
||||
"paired_different_file_count": mirror["paired_different_file_count"],
|
||||
},
|
||||
})
|
||||
if artifacts["model_checkpoint_count"] == 0:
|
||||
findings.append({
|
||||
"id": "P1-ML-LOCAL-001",
|
||||
"severity": "info",
|
||||
"title": "No local checkpoint is available in the repository checkout",
|
||||
"interpretation": "Production model truth must be verified on the mounted Tower volume.",
|
||||
})
|
||||
summary = {
|
||||
"schema_version": SCHEMA_VERSION,
|
||||
"generated_at": now(),
|
||||
"status": "findings_present" if findings else "no_static_findings",
|
||||
"finding_count": len(findings),
|
||||
"findings": findings,
|
||||
"baseline_scope": [
|
||||
"tracked repository state",
|
||||
"static code/test/migration inventory",
|
||||
"tracked mirror comparison",
|
||||
"local artifact inventory",
|
||||
"mock/fixture/placeholder/fallback triage signals",
|
||||
],
|
||||
"separate_required_evidence": [
|
||||
"Tower database/storage audit",
|
||||
"Tower CUDA/model preflight and representative inference",
|
||||
"corpus leakage/duplicate/label/time review",
|
||||
"independent human visual review",
|
||||
],
|
||||
}
|
||||
write_json(output / "repository-inventory.json", repository)
|
||||
write_json(output / "local-artifact-inventory.json", artifacts)
|
||||
write_json(output / "static-risk-signals.json", signals)
|
||||
write_json(output / "phase1-baseline-summary.json", summary)
|
||||
print(json.dumps(summary, ensure_ascii=False, indent=2, sort_keys=True))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,128 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
REPOSITORY_ROOT = Path(__file__).resolve().parents[1]
|
||||
DEFAULT_MANIFEST = (
|
||||
REPOSITORY_ROOT / "artifacts" / "evidence" / "accuracy" / "P1" / "evidence-manifest.json"
|
||||
)
|
||||
REQUIRED_DOCUMENTS = tuple(
|
||||
REPOSITORY_ROOT / "docs" / "accuracy-program" / f"{index:02d}-{name}.md"
|
||||
for index, name in (
|
||||
(0, "execution-contract"),
|
||||
(1, "system-inventory"),
|
||||
(2, "data-lineage"),
|
||||
(3, "baseline-and-gaps"),
|
||||
(4, "risk-register"),
|
||||
(5, "metric-framework"),
|
||||
(6, "implementation-roadmap"),
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def sha256_file(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as handle:
|
||||
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def verify_record(item: dict[str, Any]) -> list[str]:
|
||||
errors: list[str] = []
|
||||
relative = item.get("path")
|
||||
if not isinstance(relative, str):
|
||||
return ["manifest record has no string path"]
|
||||
path = REPOSITORY_ROOT / relative
|
||||
if not path.is_file():
|
||||
return [f"missing file: {relative}"]
|
||||
expected_size = item.get("size_bytes")
|
||||
if path.stat().st_size != expected_size:
|
||||
errors.append(
|
||||
f"size mismatch for {relative}: expected {expected_size}, got {path.stat().st_size}"
|
||||
)
|
||||
expected_hash = item.get("sha256")
|
||||
actual_hash = sha256_file(path)
|
||||
if actual_hash != expected_hash:
|
||||
errors.append(
|
||||
f"sha256 mismatch for {relative}: expected {expected_hash}, got {actual_hash}"
|
||||
)
|
||||
return errors
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Verify GeoIntel Accuracy P1 evidence and program hashes.")
|
||||
parser.add_argument("--manifest", type=Path, default=DEFAULT_MANIFEST)
|
||||
args = parser.parse_args()
|
||||
manifest_path = args.manifest.expanduser().resolve()
|
||||
if not manifest_path.is_file():
|
||||
parser.error(f"evidence manifest does not exist: {manifest_path}")
|
||||
|
||||
payload = json.loads(manifest_path.read_text(encoding="utf-8"))
|
||||
errors: list[str] = []
|
||||
evidence_records = payload.get("evidence_files")
|
||||
program_records = payload.get("program_files")
|
||||
if not isinstance(evidence_records, list) or not isinstance(program_records, list):
|
||||
errors.append("manifest must contain evidence_files and program_files arrays")
|
||||
evidence_records = []
|
||||
program_records = []
|
||||
|
||||
for item in [*evidence_records, *program_records]:
|
||||
if not isinstance(item, dict):
|
||||
errors.append("manifest file record is not an object")
|
||||
continue
|
||||
errors.extend(verify_record(item))
|
||||
|
||||
evidence_root = REPOSITORY_ROOT / str(payload.get("evidence_root") or "")
|
||||
listed = {
|
||||
str(item.get("path"))
|
||||
for item in evidence_records
|
||||
if isinstance(item, dict) and isinstance(item.get("path"), str)
|
||||
}
|
||||
current = {
|
||||
path.relative_to(REPOSITORY_ROOT).as_posix()
|
||||
for path in evidence_root.rglob("*")
|
||||
if path.is_file() and path.resolve() != manifest_path
|
||||
}
|
||||
for relative in sorted(current - listed):
|
||||
errors.append(f"unlisted evidence file: {relative}")
|
||||
for relative in sorted(listed - current):
|
||||
errors.append(f"listed evidence file no longer exists: {relative}")
|
||||
|
||||
for document in REQUIRED_DOCUMENTS:
|
||||
if not document.is_file() or document.stat().st_size == 0:
|
||||
errors.append(f"required document missing or empty: {document.relative_to(REPOSITORY_ROOT)}")
|
||||
|
||||
status_path = REPOSITORY_ROOT / "docs" / "accuracy-program" / "status.json"
|
||||
if not status_path.is_file():
|
||||
errors.append("required status.json is missing")
|
||||
else:
|
||||
status = json.loads(status_path.read_text(encoding="utf-8"))
|
||||
if status.get("phase1", {}).get("status") != "complete":
|
||||
errors.append("status.json must mark Phase 1 complete")
|
||||
if status.get("release", {}).get("status") != "blocked":
|
||||
errors.append("status.json must keep release blocked")
|
||||
if status.get("release", {}).get("promotion_allowed") is not False:
|
||||
errors.append("status.json must keep promotion disallowed")
|
||||
if status.get("scope", {}).get("national_building_validation") is not False:
|
||||
errors.append("status.json must not claim national building validation")
|
||||
|
||||
summary = {
|
||||
"schema_version": 1,
|
||||
"status": "passed" if not errors else "failed",
|
||||
"manifest": manifest_path.relative_to(REPOSITORY_ROOT).as_posix(),
|
||||
"evidence_files_checked": len(evidence_records),
|
||||
"program_files_checked": len(program_records),
|
||||
"errors": errors,
|
||||
}
|
||||
print(json.dumps(summary, indent=2, sort_keys=True))
|
||||
return 0 if not errors else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in New Issue
Block a user