Files
geointel/scripts/collect_accuracy_phase1_inference_smoke.py
T

182 lines
6.6 KiB
Python

from __future__ import annotations
import argparse
from collections import Counter
from datetime import datetime, timezone
import hashlib
import json
from pathlib import Path
import random
import sys
from time import perf_counter
from typing import Any
REPOSITORY_ROOT = Path(__file__).resolve().parents[1]
BACKEND_ROOT = REPOSITORY_ROOT / "backend"
if str(BACKEND_ROOT) not in sys.path:
sys.path.insert(0, str(BACKEND_ROOT))
from app.core.config import Settings # noqa: E402
from app.services.yolo_adapter import YoloDetectionAdapter # noqa: E402
def _sha256(path: Path) -> str:
digest = hashlib.sha256()
with path.open("rb") as handle:
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
digest.update(chunk)
return digest.hexdigest()
def _raster_metadata(path: Path) -> dict[str, Any]:
try:
import rasterio
except ImportError:
return {"available": False, "reason": "rasterio_not_installed"}
with rasterio.open(path) as dataset:
return {
"available": True,
"bounds": [float(value) for value in dataset.bounds],
"count": int(dataset.count),
"crs": str(dataset.crs) if dataset.crs else None,
"dtypes": list(dataset.dtypes),
"height": int(dataset.height),
"nodata": dataset.nodata,
"transform": [float(value) for value in dataset.transform],
"width": int(dataset.width),
}
def _summarize_detections(detections: list[dict[str, Any]]) -> dict[str, Any]:
confidences = [float(item["confidence"]) for item in detections]
class_counts = Counter(str(item["class_name"]) for item in detections)
return {
"count": len(detections),
"class_counts": dict(sorted(class_counts.items())),
"confidence": {
"minimum": min(confidences) if confidences else None,
"maximum": max(confidences) if confidences else None,
"mean": sum(confidences) / len(confidences) if confidences else None,
},
"sample": detections[:10],
}
def main() -> int:
parser = argparse.ArgumentParser(
description=(
"Run one read-only production-adapter inference and emit forensic JSON. "
"This proves runtime execution only; it does not establish model accuracy."
)
)
parser.add_argument("--model-path", required=True)
parser.add_argument("--tile-path", required=True)
parser.add_argument("--manifest-path")
parser.add_argument("--confidence", type=float, default=0.5)
parser.add_argument("--image-size", type=int, default=640)
parser.add_argument("--max-detections", type=int, default=1000)
parser.add_argument("--device", default="cuda:0")
parser.add_argument("--seed", type=int, default=20260801)
args = parser.parse_args()
model_path = Path(args.model_path).expanduser().resolve()
tile_path = Path(args.tile_path).expanduser().resolve()
manifest_path = Path(args.manifest_path).expanduser().resolve() if args.manifest_path else None
for label, path in (("model", model_path), ("tile", tile_path)):
if not path.is_file():
parser.error(f"{label} path is not an existing file: {path}")
if manifest_path is not None and not manifest_path.is_file():
parser.error(f"manifest path is not an existing file: {manifest_path}")
if not 0.0 <= args.confidence <= 1.0:
parser.error("--confidence must be between 0 and 1")
import numpy as np
import torch
import ultralytics
random.seed(args.seed)
np.random.seed(args.seed)
torch.manual_seed(args.seed)
if torch.cuda.is_available():
torch.cuda.manual_seed_all(args.seed)
torch.cuda.reset_peak_memory_stats()
torch.use_deterministic_algorithms(True, warn_only=True)
settings = Settings().model_copy(
update={
"yolo_enabled": True,
"yolo_model_path": str(model_path),
"yolo_device": args.device,
"yolo_require_cuda": True,
"yolo_image_size": args.image_size,
"yolo_max_detections": args.max_detections,
}
)
adapter = YoloDetectionAdapter(settings)
adapter.validate_runtime()
started = perf_counter()
model = adapter.load_model(model_path)
model_loaded = perf_counter()
detections = adapter.predict_tile(model, tile_path, args.confidence)
if torch.cuda.is_available():
torch.cuda.synchronize()
finished = perf_counter()
device_index = torch.cuda.current_device() if torch.cuda.is_available() else None
payload = {
"schema_version": 1,
"captured_at": datetime.now(timezone.utc).isoformat(),
"status": "passed",
"claim_boundary": (
"One production-adapter inference completed on one existing tile. "
"No accuracy, calibration, geographic-generalization, or release claim follows from this smoke."
),
"read_only": True,
"configuration": {
"confidence": args.confidence,
"device": args.device,
"image_size": args.image_size,
"max_detections": args.max_detections,
"seed": args.seed,
"deterministic_algorithms": True,
},
"model": {
"path": str(model_path),
"sha256": _sha256(model_path),
"size_bytes": model_path.stat().st_size,
},
"input": {
"tile_path": str(tile_path),
"tile_sha256": _sha256(tile_path),
"tile_size_bytes": tile_path.stat().st_size,
"manifest_path": str(manifest_path) if manifest_path else None,
"manifest_sha256": _sha256(manifest_path) if manifest_path else None,
"raster": _raster_metadata(tile_path),
},
"runtime": {
"python": sys.version,
"torch": torch.__version__,
"ultralytics": ultralytics.__version__,
"cuda_available": torch.cuda.is_available(),
"cuda_runtime": torch.version.cuda,
"cuda_device_index": device_index,
"cuda_device_name": torch.cuda.get_device_name(device_index) if device_index is not None else None,
"cuda_peak_memory_bytes": torch.cuda.max_memory_allocated() if torch.cuda.is_available() else None,
},
"timing_seconds": {
"model_load": model_loaded - started,
"inference": finished - model_loaded,
"total": finished - started,
},
"output": _summarize_detections(detections),
}
print(json.dumps(payload, indent=2, sort_keys=True))
return 0
if __name__ == "__main__":
raise SystemExit(main())