geointel/scripts/build_fixed_threshold_evidence_portfolio_inputs.py

from __future__ import annotations

import argparse
import json
from pathlib import Path
from typing import Any


def parse_args() -> argparse.Namespace:
    parser = argparse.ArgumentParser(
        description="Build comparable fixed-threshold inputs for a GeoIntel calibration evidence portfolio."
    )
    parser.add_argument("--multi-sample-summary", required=True, type=Path)
    parser.add_argument("--threshold", required=True, type=float)
    parser.add_argument("--model-asset-id")
    parser.add_argument("--model-sha256")
    parser.add_argument("--tile-size", type=int)
    parser.add_argument("--tile-overlap", type=int)
    parser.add_argument("--output-dir", required=True, type=Path)
    return parser.parse_args()


def load_json(path: Path) -> dict[str, Any]:
    if not path.is_file():
        raise SystemExit(f"JSON input is not readable: {path}")
    payload = json.loads(path.read_text(encoding="utf-8-sig"))
    if not isinstance(payload, dict):
        raise SystemExit(f"JSON input must be an object: {path}")
    return payload


def resolve_summary_path(raw: str, multi_summary_path: Path) -> Path:
    path = Path(raw).expanduser()
    candidates = [path]
    if not path.is_absolute():
        candidates.extend((multi_summary_path.parent / path, Path.cwd() / path))
    for candidate in candidates:
        if candidate.is_file():
            return candidate.resolve()
    raise SystemExit(f"Sample summary is not readable: {raw}")


def matches_run(
    item: dict[str, Any],
    *,
    threshold: float,
    model_asset_id: str | None,
    tile_size: int | None,
    tile_overlap: int | None,
) -> bool:
    try:
        item_threshold = float(item.get("threshold"))
    except (TypeError, ValueError):
        return False
    if abs(item_threshold - threshold) > 1e-9:
        return False
    if model_asset_id and item.get("model_asset_id") != model_asset_id:
        return False
    if tile_size is not None and item.get("tile_size") != tile_size:
        return False
    if tile_overlap is not None and item.get("tile_overlap") != tile_overlap:
        return False
    return True


def build_inputs(args: argparse.Namespace) -> Path:
    multi_summary_path = args.multi_sample_summary.expanduser().resolve()
    multi_summary = load_json(multi_summary_path)
    sample_summaries = multi_summary.get("sample_summaries") or []
    if not isinstance(sample_summaries, list) or not sample_summaries:
        raise SystemExit("Multi-sample summary has no sample_summaries")

    output_dir = args.output_dir.expanduser().resolve()
    samples_dir = output_dir / "samples"
    samples_dir.mkdir(parents=True, exist_ok=True)
    manifest_samples: list[dict[str, Any]] = []
    selected_model_ids: set[str] = set()

    for sample in sample_summaries:
        if not isinstance(sample, dict):
            raise SystemExit("Every sample summary entry must be an object")
        sample_slug = str(sample.get("sample_slug") or "").strip().lower()
        if not sample_slug:
            raise SystemExit("Sample summary entry is missing sample_slug")
        source_summary_path = resolve_summary_path(
            str(sample.get("summary_path") or ""), multi_summary_path
        )
        source_summary = load_json(source_summary_path)
        items = source_summary.get("items") or []
        matches = [
            item
            for item in items
            if isinstance(item, dict)
            and matches_run(
                item,
                threshold=args.threshold,
                model_asset_id=args.model_asset_id,
                tile_size=args.tile_size,
                tile_overlap=args.tile_overlap,
            )
        ]
        if len(matches) != 1:
            raise SystemExit(
                f"Expected exactly one fixed-threshold run for {sample_slug}; found {len(matches)}"
            )
        selected = dict(matches[0])
        selected_model_id = str(selected.get("model_asset_id") or "").strip()
        if not selected_model_id:
            raise SystemExit(f"Selected run for {sample_slug} has no model_asset_id")
        selected_model_ids.add(selected_model_id)

        filtered_summary = dict(source_summary)
        filtered_summary.update(
            {
                "items": [selected],
                "best_by_score": selected,
                "best_by_recall": selected,
                "best_by_precision": selected,
                "fixed_threshold": args.threshold,
                "source_summary_path": str(source_summary_path),
            }
        )
        sample_dir = samples_dir / sample_slug
        sample_dir.mkdir(parents=True, exist_ok=True)
        filtered_path = sample_dir / "quality_matrix_summary.json"
        filtered_path.write_text(
            json.dumps(filtered_summary, indent=2, sort_keys=True), encoding="utf-8"
        )
        manifest_samples.append(
            {
                "sample_slug": sample_slug,
                "aoi_label": sample_slug.replace("_", " ").title(),
                "summary_path": str(filtered_path),
                "operator_notes": (
                    f"Fixed threshold {args.threshold:g}; source {source_summary_path}."
                ),
            }
        )

    if len(selected_model_ids) != 1:
        raise SystemExit(
            f"Fixed-threshold runs contain multiple model assets: {sorted(selected_model_ids)}"
        )
    selected_model_id = next(iter(selected_model_ids))
    manifest = {
        "portfolio_name": (
            f"GeoIntel fixed-threshold false-negative review: {selected_model_id} @ {args.threshold:g}"
        ),
        "model_asset_id": selected_model_id,
        "model_sha256": args.model_sha256,
        "fixed_threshold": args.threshold,
        "notes": "Comparable per-AOI persisted QA evidence; no inference is run by this input builder.",
        "samples": manifest_samples,
    }
    manifest_path = output_dir / "calibration-evidence-portfolio-manifest.json"
    manifest_path.write_text(
        json.dumps(manifest, indent=2, sort_keys=True), encoding="utf-8"
    )
    return manifest_path


def main() -> None:
    manifest_path = build_inputs(parse_args())
    print(f"Fixed-threshold portfolio manifest: {manifest_path}")


if __name__ == "__main__":
    main()