from __future__ import annotations import argparse import json from pathlib import Path from typing import Any def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser( description="Build comparable fixed-threshold inputs for a GeoIntel calibration evidence portfolio." ) parser.add_argument("--multi-sample-summary", required=True, type=Path) parser.add_argument("--threshold", required=True, type=float) parser.add_argument("--model-asset-id") parser.add_argument("--model-sha256") parser.add_argument("--tile-size", type=int) parser.add_argument("--tile-overlap", type=int) parser.add_argument("--output-dir", required=True, type=Path) return parser.parse_args() def load_json(path: Path) -> dict[str, Any]: if not path.is_file(): raise SystemExit(f"JSON input is not readable: {path}") payload = json.loads(path.read_text(encoding="utf-8-sig")) if not isinstance(payload, dict): raise SystemExit(f"JSON input must be an object: {path}") return payload def resolve_summary_path(raw: str, multi_summary_path: Path) -> Path: path = Path(raw).expanduser() candidates = [path] if not path.is_absolute(): candidates.extend((multi_summary_path.parent / path, Path.cwd() / path)) for candidate in candidates: if candidate.is_file(): return candidate.resolve() raise SystemExit(f"Sample summary is not readable: {raw}") def matches_run( item: dict[str, Any], *, threshold: float, model_asset_id: str | None, tile_size: int | None, tile_overlap: int | None, ) -> bool: try: item_threshold = float(item.get("threshold")) except (TypeError, ValueError): return False if abs(item_threshold - threshold) > 1e-9: return False if model_asset_id and item.get("model_asset_id") != model_asset_id: return False if tile_size is not None and item.get("tile_size") != tile_size: return False if tile_overlap is not None and item.get("tile_overlap") != tile_overlap: return False return True def build_inputs(args: argparse.Namespace) -> Path: multi_summary_path = args.multi_sample_summary.expanduser().resolve() multi_summary = load_json(multi_summary_path) sample_summaries = multi_summary.get("sample_summaries") or [] if not isinstance(sample_summaries, list) or not sample_summaries: raise SystemExit("Multi-sample summary has no sample_summaries") output_dir = args.output_dir.expanduser().resolve() samples_dir = output_dir / "samples" samples_dir.mkdir(parents=True, exist_ok=True) manifest_samples: list[dict[str, Any]] = [] selected_model_ids: set[str] = set() for sample in sample_summaries: if not isinstance(sample, dict): raise SystemExit("Every sample summary entry must be an object") sample_slug = str(sample.get("sample_slug") or "").strip().lower() if not sample_slug: raise SystemExit("Sample summary entry is missing sample_slug") source_summary_path = resolve_summary_path( str(sample.get("summary_path") or ""), multi_summary_path ) source_summary = load_json(source_summary_path) items = source_summary.get("items") or [] matches = [ item for item in items if isinstance(item, dict) and matches_run( item, threshold=args.threshold, model_asset_id=args.model_asset_id, tile_size=args.tile_size, tile_overlap=args.tile_overlap, ) ] if len(matches) != 1: raise SystemExit( f"Expected exactly one fixed-threshold run for {sample_slug}; found {len(matches)}" ) selected = dict(matches[0]) selected_model_id = str(selected.get("model_asset_id") or "").strip() if not selected_model_id: raise SystemExit(f"Selected run for {sample_slug} has no model_asset_id") selected_model_ids.add(selected_model_id) filtered_summary = dict(source_summary) filtered_summary.update( { "items": [selected], "best_by_score": selected, "best_by_recall": selected, "best_by_precision": selected, "fixed_threshold": args.threshold, "source_summary_path": str(source_summary_path), } ) sample_dir = samples_dir / sample_slug sample_dir.mkdir(parents=True, exist_ok=True) filtered_path = sample_dir / "quality_matrix_summary.json" filtered_path.write_text( json.dumps(filtered_summary, indent=2, sort_keys=True), encoding="utf-8" ) manifest_samples.append( { "sample_slug": sample_slug, "aoi_label": sample_slug.replace("_", " ").title(), "summary_path": str(filtered_path), "operator_notes": ( f"Fixed threshold {args.threshold:g}; source {source_summary_path}." ), } ) if len(selected_model_ids) != 1: raise SystemExit( f"Fixed-threshold runs contain multiple model assets: {sorted(selected_model_ids)}" ) selected_model_id = next(iter(selected_model_ids)) manifest = { "portfolio_name": ( f"GeoIntel fixed-threshold false-negative review: {selected_model_id} @ {args.threshold:g}" ), "model_asset_id": selected_model_id, "model_sha256": args.model_sha256, "fixed_threshold": args.threshold, "notes": "Comparable per-AOI persisted QA evidence; no inference is run by this input builder.", "samples": manifest_samples, } manifest_path = output_dir / "calibration-evidence-portfolio-manifest.json" manifest_path.write_text( json.dumps(manifest, indent=2, sort_keys=True), encoding="utf-8" ) return manifest_path def main() -> None: manifest_path = build_inputs(parse_args()) print(f"Fixed-threshold portfolio manifest: {manifest_path}") if __name__ == "__main__": main()