geointel/scripts/collect_accuracy_phase1_inference_smoke.py

from __future__ import annotations

import argparse
from collections import Counter
from datetime import datetime, timezone
import hashlib
import json
from pathlib import Path
import random
import sys
from time import perf_counter
from typing import Any


REPOSITORY_ROOT = Path(__file__).resolve().parents[1]
BACKEND_ROOT = REPOSITORY_ROOT / "backend"
if str(BACKEND_ROOT) not in sys.path:
    sys.path.insert(0, str(BACKEND_ROOT))

from app.core.config import Settings  # noqa: E402
from app.services.yolo_adapter import YoloDetectionAdapter  # noqa: E402


def _sha256(path: Path) -> str:
    digest = hashlib.sha256()
    with path.open("rb") as handle:
        for chunk in iter(lambda: handle.read(1024 * 1024), b""):
            digest.update(chunk)
    return digest.hexdigest()


def _raster_metadata(path: Path) -> dict[str, Any]:
    try:
        import rasterio
    except ImportError:
        return {"available": False, "reason": "rasterio_not_installed"}

    with rasterio.open(path) as dataset:
        return {
            "available": True,
            "bounds": [float(value) for value in dataset.bounds],
            "count": int(dataset.count),
            "crs": str(dataset.crs) if dataset.crs else None,
            "dtypes": list(dataset.dtypes),
            "height": int(dataset.height),
            "nodata": dataset.nodata,
            "transform": [float(value) for value in dataset.transform],
            "width": int(dataset.width),
        }


def _summarize_detections(detections: list[dict[str, Any]]) -> dict[str, Any]:
    confidences = [float(item["confidence"]) for item in detections]
    class_counts = Counter(str(item["class_name"]) for item in detections)
    return {
        "count": len(detections),
        "class_counts": dict(sorted(class_counts.items())),
        "confidence": {
            "minimum": min(confidences) if confidences else None,
            "maximum": max(confidences) if confidences else None,
            "mean": sum(confidences) / len(confidences) if confidences else None,
        },
        "sample": detections[:10],
    }


def main() -> int:
    parser = argparse.ArgumentParser(
        description=(
            "Run one read-only production-adapter inference and emit forensic JSON. "
            "This proves runtime execution only; it does not establish model accuracy."
        )
    )
    parser.add_argument("--model-path", required=True)
    parser.add_argument("--tile-path", required=True)
    parser.add_argument("--manifest-path")
    parser.add_argument("--confidence", type=float, default=0.5)
    parser.add_argument("--image-size", type=int, default=640)
    parser.add_argument("--max-detections", type=int, default=1000)
    parser.add_argument("--device", default="cuda:0")
    parser.add_argument("--seed", type=int, default=20260801)
    args = parser.parse_args()

    model_path = Path(args.model_path).expanduser().resolve()
    tile_path = Path(args.tile_path).expanduser().resolve()
    manifest_path = Path(args.manifest_path).expanduser().resolve() if args.manifest_path else None
    for label, path in (("model", model_path), ("tile", tile_path)):
        if not path.is_file():
            parser.error(f"{label} path is not an existing file: {path}")
    if manifest_path is not None and not manifest_path.is_file():
        parser.error(f"manifest path is not an existing file: {manifest_path}")
    if not 0.0 <= args.confidence <= 1.0:
        parser.error("--confidence must be between 0 and 1")

    import numpy as np
    import torch
    import ultralytics

    random.seed(args.seed)
    np.random.seed(args.seed)
    torch.manual_seed(args.seed)
    if torch.cuda.is_available():
        torch.cuda.manual_seed_all(args.seed)
        torch.cuda.reset_peak_memory_stats()
    torch.use_deterministic_algorithms(True, warn_only=True)

    settings = Settings().model_copy(
        update={
            "yolo_enabled": True,
            "yolo_model_path": str(model_path),
            "yolo_device": args.device,
            "yolo_require_cuda": True,
            "yolo_image_size": args.image_size,
            "yolo_max_detections": args.max_detections,
        }
    )
    adapter = YoloDetectionAdapter(settings)
    adapter.validate_runtime()

    started = perf_counter()
    model = adapter.load_model(model_path)
    model_loaded = perf_counter()
    detections = adapter.predict_tile(model, tile_path, args.confidence)
    if torch.cuda.is_available():
        torch.cuda.synchronize()
    finished = perf_counter()

    device_index = torch.cuda.current_device() if torch.cuda.is_available() else None
    payload = {
        "schema_version": 1,
        "captured_at": datetime.now(timezone.utc).isoformat(),
        "status": "passed",
        "claim_boundary": (
            "One production-adapter inference completed on one existing tile. "
            "No accuracy, calibration, geographic-generalization, or release claim follows from this smoke."
        ),
        "read_only": True,
        "configuration": {
            "confidence": args.confidence,
            "device": args.device,
            "image_size": args.image_size,
            "max_detections": args.max_detections,
            "seed": args.seed,
            "deterministic_algorithms": True,
        },
        "model": {
            "path": str(model_path),
            "sha256": _sha256(model_path),
            "size_bytes": model_path.stat().st_size,
        },
        "input": {
            "tile_path": str(tile_path),
            "tile_sha256": _sha256(tile_path),
            "tile_size_bytes": tile_path.stat().st_size,
            "manifest_path": str(manifest_path) if manifest_path else None,
            "manifest_sha256": _sha256(manifest_path) if manifest_path else None,
            "raster": _raster_metadata(tile_path),
        },
        "runtime": {
            "python": sys.version,
            "torch": torch.__version__,
            "ultralytics": ultralytics.__version__,
            "cuda_available": torch.cuda.is_available(),
            "cuda_runtime": torch.version.cuda,
            "cuda_device_index": device_index,
            "cuda_device_name": torch.cuda.get_device_name(device_index) if device_index is not None else None,
            "cuda_peak_memory_bytes": torch.cuda.max_memory_allocated() if torch.cuda.is_available() else None,
        },
        "timing_seconds": {
            "model_load": model_loaded - started,
            "inference": finished - model_loaded,
            "total": finished - started,
        },
        "output": _summarize_detections(detections),
    }
    print(json.dumps(payload, indent=2, sort_keys=True))
    return 0


if __name__ == "__main__":
    raise SystemExit(main())