audit: establish accuracy phase 1 baseline
This commit is contained in:
Jens
@@ -0,0 +1,181 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
from collections import Counter
|
||||
from datetime import datetime, timezone
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
import random
|
||||
import sys
|
||||
from time import perf_counter
|
||||
from typing import Any
|
||||
|
||||
|
||||
REPOSITORY_ROOT = Path(__file__).resolve().parents[1]
|
||||
BACKEND_ROOT = REPOSITORY_ROOT / "backend"
|
||||
if str(BACKEND_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(BACKEND_ROOT))
|
||||
|
||||
from app.core.config import Settings # noqa: E402
|
||||
from app.services.yolo_adapter import YoloDetectionAdapter # noqa: E402
|
||||
|
||||
|
||||
def _sha256(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as handle:
|
||||
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def _raster_metadata(path: Path) -> dict[str, Any]:
|
||||
try:
|
||||
import rasterio
|
||||
except ImportError:
|
||||
return {"available": False, "reason": "rasterio_not_installed"}
|
||||
|
||||
with rasterio.open(path) as dataset:
|
||||
return {
|
||||
"available": True,
|
||||
"bounds": [float(value) for value in dataset.bounds],
|
||||
"count": int(dataset.count),
|
||||
"crs": str(dataset.crs) if dataset.crs else None,
|
||||
"dtypes": list(dataset.dtypes),
|
||||
"height": int(dataset.height),
|
||||
"nodata": dataset.nodata,
|
||||
"transform": [float(value) for value in dataset.transform],
|
||||
"width": int(dataset.width),
|
||||
}
|
||||
|
||||
|
||||
def _summarize_detections(detections: list[dict[str, Any]]) -> dict[str, Any]:
|
||||
confidences = [float(item["confidence"]) for item in detections]
|
||||
class_counts = Counter(str(item["class_name"]) for item in detections)
|
||||
return {
|
||||
"count": len(detections),
|
||||
"class_counts": dict(sorted(class_counts.items())),
|
||||
"confidence": {
|
||||
"minimum": min(confidences) if confidences else None,
|
||||
"maximum": max(confidences) if confidences else None,
|
||||
"mean": sum(confidences) / len(confidences) if confidences else None,
|
||||
},
|
||||
"sample": detections[:10],
|
||||
}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(
|
||||
description=(
|
||||
"Run one read-only production-adapter inference and emit forensic JSON. "
|
||||
"This proves runtime execution only; it does not establish model accuracy."
|
||||
)
|
||||
)
|
||||
parser.add_argument("--model-path", required=True)
|
||||
parser.add_argument("--tile-path", required=True)
|
||||
parser.add_argument("--manifest-path")
|
||||
parser.add_argument("--confidence", type=float, default=0.5)
|
||||
parser.add_argument("--image-size", type=int, default=640)
|
||||
parser.add_argument("--max-detections", type=int, default=1000)
|
||||
parser.add_argument("--device", default="cuda:0")
|
||||
parser.add_argument("--seed", type=int, default=20260801)
|
||||
args = parser.parse_args()
|
||||
|
||||
model_path = Path(args.model_path).expanduser().resolve()
|
||||
tile_path = Path(args.tile_path).expanduser().resolve()
|
||||
manifest_path = Path(args.manifest_path).expanduser().resolve() if args.manifest_path else None
|
||||
for label, path in (("model", model_path), ("tile", tile_path)):
|
||||
if not path.is_file():
|
||||
parser.error(f"{label} path is not an existing file: {path}")
|
||||
if manifest_path is not None and not manifest_path.is_file():
|
||||
parser.error(f"manifest path is not an existing file: {manifest_path}")
|
||||
if not 0.0 <= args.confidence <= 1.0:
|
||||
parser.error("--confidence must be between 0 and 1")
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
import ultralytics
|
||||
|
||||
random.seed(args.seed)
|
||||
np.random.seed(args.seed)
|
||||
torch.manual_seed(args.seed)
|
||||
if torch.cuda.is_available():
|
||||
torch.cuda.manual_seed_all(args.seed)
|
||||
torch.cuda.reset_peak_memory_stats()
|
||||
torch.use_deterministic_algorithms(True, warn_only=True)
|
||||
|
||||
settings = Settings().model_copy(
|
||||
update={
|
||||
"yolo_enabled": True,
|
||||
"yolo_model_path": str(model_path),
|
||||
"yolo_device": args.device,
|
||||
"yolo_require_cuda": True,
|
||||
"yolo_image_size": args.image_size,
|
||||
"yolo_max_detections": args.max_detections,
|
||||
}
|
||||
)
|
||||
adapter = YoloDetectionAdapter(settings)
|
||||
adapter.validate_runtime()
|
||||
|
||||
started = perf_counter()
|
||||
model = adapter.load_model(model_path)
|
||||
model_loaded = perf_counter()
|
||||
detections = adapter.predict_tile(model, tile_path, args.confidence)
|
||||
if torch.cuda.is_available():
|
||||
torch.cuda.synchronize()
|
||||
finished = perf_counter()
|
||||
|
||||
device_index = torch.cuda.current_device() if torch.cuda.is_available() else None
|
||||
payload = {
|
||||
"schema_version": 1,
|
||||
"captured_at": datetime.now(timezone.utc).isoformat(),
|
||||
"status": "passed",
|
||||
"claim_boundary": (
|
||||
"One production-adapter inference completed on one existing tile. "
|
||||
"No accuracy, calibration, geographic-generalization, or release claim follows from this smoke."
|
||||
),
|
||||
"read_only": True,
|
||||
"configuration": {
|
||||
"confidence": args.confidence,
|
||||
"device": args.device,
|
||||
"image_size": args.image_size,
|
||||
"max_detections": args.max_detections,
|
||||
"seed": args.seed,
|
||||
"deterministic_algorithms": True,
|
||||
},
|
||||
"model": {
|
||||
"path": str(model_path),
|
||||
"sha256": _sha256(model_path),
|
||||
"size_bytes": model_path.stat().st_size,
|
||||
},
|
||||
"input": {
|
||||
"tile_path": str(tile_path),
|
||||
"tile_sha256": _sha256(tile_path),
|
||||
"tile_size_bytes": tile_path.stat().st_size,
|
||||
"manifest_path": str(manifest_path) if manifest_path else None,
|
||||
"manifest_sha256": _sha256(manifest_path) if manifest_path else None,
|
||||
"raster": _raster_metadata(tile_path),
|
||||
},
|
||||
"runtime": {
|
||||
"python": sys.version,
|
||||
"torch": torch.__version__,
|
||||
"ultralytics": ultralytics.__version__,
|
||||
"cuda_available": torch.cuda.is_available(),
|
||||
"cuda_runtime": torch.version.cuda,
|
||||
"cuda_device_index": device_index,
|
||||
"cuda_device_name": torch.cuda.get_device_name(device_index) if device_index is not None else None,
|
||||
"cuda_peak_memory_bytes": torch.cuda.max_memory_allocated() if torch.cuda.is_available() else None,
|
||||
},
|
||||
"timing_seconds": {
|
||||
"model_load": model_loaded - started,
|
||||
"inference": finished - model_loaded,
|
||||
"total": finished - started,
|
||||
},
|
||||
"output": _summarize_detections(detections),
|
||||
}
|
||||
print(json.dumps(payload, indent=2, sort_keys=True))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in New Issue
Block a user