297 lines
11 KiB
Bash
297 lines
11 KiB
Bash
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
|
|
usage() {
|
|
cat >&2 <<'EOF'
|
|
Usage:
|
|
bash scripts/assemble_detection_calibration_evidence_portfolio.sh [base_url] /path/to/calibration-evidence-portfolio-manifest.json
|
|
|
|
Required manifest shape:
|
|
{
|
|
"portfolio_name": "Kempen building model calibration",
|
|
"model_asset_id": "geointel-building-yolov8s-hardneg160r4e50-pt",
|
|
"model_sha256": "optional",
|
|
"notes": "optional operator notes",
|
|
"samples": [
|
|
{
|
|
"sample_slug": "geel",
|
|
"aoi_label": "Geel center",
|
|
"summary_path": "/path/to/detection-calibration-summary.json",
|
|
"operator_notes": "optional AOI notes"
|
|
}
|
|
]
|
|
}
|
|
|
|
Optional environment:
|
|
CALIBRATION_PORTFOLIO_OUTPUT_DIR Output directory, default: artifacts/detection-calibration-portfolio/<timestamp>.
|
|
CALIBRATION_EVIDENCE_MODE all or best, default: all. Forwarded to export_detection_calibration_evidence.sh.
|
|
|
|
This is operator evidence tooling only. It does not run inference, create QA
|
|
checks, mutate application data, download models or fetch providers.
|
|
EOF
|
|
}
|
|
|
|
if [ "${1:-}" = "-h" ] || [ "${1:-}" = "--help" ]; then
|
|
usage
|
|
exit 0
|
|
fi
|
|
|
|
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
|
cd "$ROOT"
|
|
|
|
BASE_URL="${1:-${GE_INTEL_BASE_URL:-http://localhost:1202}}"
|
|
MANIFEST_PATH="${2:-${CALIBRATION_PORTFOLIO_MANIFEST_PATH:-}}"
|
|
CALIBRATION_EVIDENCE_MODE="${CALIBRATION_EVIDENCE_MODE:-all}"
|
|
CURL_BIN="${CURL_BIN:-curl}"
|
|
timestamp="$(date -u +%Y%m%dT%H%M%SZ)"
|
|
OUTPUT_DIR="${CALIBRATION_PORTFOLIO_OUTPUT_DIR:-${ROOT}/artifacts/detection-calibration-portfolio/${timestamp}}"
|
|
|
|
if [ -z "$MANIFEST_PATH" ]; then
|
|
usage
|
|
exit 2
|
|
fi
|
|
|
|
case "${CALIBRATION_EVIDENCE_MODE}" in
|
|
all|best) ;;
|
|
*)
|
|
echo "CALIBRATION_EVIDENCE_MODE must be 'all' or 'best'" >&2
|
|
exit 2
|
|
;;
|
|
esac
|
|
|
|
if command -v cygpath >/dev/null 2>&1 && printf '%s' "$OUTPUT_DIR" | grep -Eq '^[A-Za-z]:\\'; then
|
|
OUTPUT_DIR="$(cygpath -u "$OUTPUT_DIR")"
|
|
fi
|
|
|
|
if [ -n "${PYTHON_BIN:-}" ]; then
|
|
PYTHON_BIN="${PYTHON_BIN}"
|
|
else
|
|
PYTHON_BIN=""
|
|
for candidate in python3 python.exe python; do
|
|
if command -v "${candidate}" >/dev/null 2>&1 && "${candidate}" -c "import json, pathlib, shutil, sys" >/dev/null 2>&1; then
|
|
PYTHON_BIN="${candidate}"
|
|
break
|
|
fi
|
|
done
|
|
fi
|
|
|
|
if [ -z "${PYTHON_BIN}" ]; then
|
|
echo "A Python interpreter is required for portfolio assembly" >&2
|
|
exit 1
|
|
fi
|
|
|
|
mkdir -p "$OUTPUT_DIR"
|
|
SAMPLE_REQUESTS="${OUTPUT_DIR}/portfolio_sample_requests.tsv"
|
|
PORTFOLIO_META="${OUTPUT_DIR}/portfolio_input_metadata.json"
|
|
|
|
"${PYTHON_BIN}" - "$MANIFEST_PATH" "$OUTPUT_DIR" "$SAMPLE_REQUESTS" "$PORTFOLIO_META" <<'PY'
|
|
import json
|
|
import os
|
|
import re
|
|
import shutil
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
manifest_path_raw, output_dir_raw, requests_path_raw, metadata_path_raw = sys.argv[1:5]
|
|
|
|
|
|
def as_path(raw: str) -> Path:
|
|
return Path(raw).expanduser()
|
|
|
|
|
|
manifest_path = as_path(manifest_path_raw)
|
|
if not manifest_path.is_file():
|
|
raise SystemExit(f"Manifest is not readable: {manifest_path_raw}")
|
|
|
|
output_dir = as_path(output_dir_raw)
|
|
samples_dir = output_dir / "samples"
|
|
samples_dir.mkdir(parents=True, exist_ok=True)
|
|
|
|
manifest = json.loads(manifest_path.read_text(encoding="utf-8-sig"))
|
|
samples = manifest.get("samples") or []
|
|
if not isinstance(samples, list) or not samples:
|
|
raise SystemExit("Portfolio manifest must contain at least one sample")
|
|
|
|
|
|
def safe_slug(raw: str) -> str:
|
|
slug = re.sub(r"[^A-Za-z0-9_.-]+", "-", str(raw or "").strip()).strip("-._").lower()
|
|
if not slug:
|
|
raise SystemExit("Every portfolio sample needs a non-empty sample_slug")
|
|
return slug
|
|
|
|
|
|
request_rows = []
|
|
normalized_samples = []
|
|
seen_slugs = set()
|
|
for sample in samples:
|
|
if not isinstance(sample, dict):
|
|
raise SystemExit("Every portfolio sample must be an object")
|
|
sample_slug = safe_slug(sample.get("sample_slug"))
|
|
if sample_slug in seen_slugs:
|
|
raise SystemExit(f"Duplicate portfolio sample_slug: {sample_slug}")
|
|
seen_slugs.add(sample_slug)
|
|
summary_path = as_path(str(sample.get("summary_path") or ""))
|
|
if not summary_path.is_file():
|
|
raise SystemExit(f"Sample summary_path is not readable for {sample_slug}: {summary_path}")
|
|
sample_dir = samples_dir / sample_slug
|
|
evidence_dir = sample_dir / "evidence"
|
|
sample_dir.mkdir(parents=True, exist_ok=True)
|
|
evidence_dir.mkdir(parents=True, exist_ok=True)
|
|
copied_summary = sample_dir / summary_path.name
|
|
shutil.copyfile(summary_path, copied_summary)
|
|
normalized = {
|
|
"sample_slug": sample_slug,
|
|
"aoi_label": sample.get("aoi_label") or sample_slug,
|
|
"operator_notes": sample.get("operator_notes") or "",
|
|
"source_summary_path": str(summary_path),
|
|
"copied_summary_path": str(copied_summary),
|
|
"evidence_dir": str(evidence_dir),
|
|
}
|
|
normalized_samples.append(normalized)
|
|
request_rows.append((sample_slug, str(copied_summary), str(evidence_dir)))
|
|
|
|
metadata = {
|
|
"portfolio_name": manifest.get("portfolio_name") or "GeoIntel detection calibration evidence portfolio",
|
|
"model_asset_id": manifest.get("model_asset_id"),
|
|
"model_sha256": manifest.get("model_sha256"),
|
|
"notes": manifest.get("notes") or "",
|
|
"manifest_path": str(manifest_path),
|
|
"samples": normalized_samples,
|
|
}
|
|
Path(metadata_path_raw).write_text(json.dumps(metadata, indent=2, sort_keys=True), encoding="utf-8")
|
|
with Path(requests_path_raw).open("w", encoding="utf-8") as handle:
|
|
for row in request_rows:
|
|
handle.write("\t".join(row) + "\n")
|
|
PY
|
|
|
|
echo "== GeoIntel detection calibration evidence portfolio =="
|
|
echo "Base URL: ${BASE_URL}"
|
|
echo "Manifest: ${MANIFEST_PATH}"
|
|
echo "Mode: ${CALIBRATION_EVIDENCE_MODE}"
|
|
echo "Output: ${OUTPUT_DIR}"
|
|
|
|
while IFS=$'\t' read -r sample_slug summary_path evidence_dir; do
|
|
echo "-- Portfolio sample ${sample_slug} --"
|
|
CALIBRATION_EVIDENCE_DIR="${evidence_dir}" \
|
|
CALIBRATION_EVIDENCE_MODE="${CALIBRATION_EVIDENCE_MODE}" \
|
|
CURL_BIN="${CURL_BIN:-curl}" \
|
|
bash scripts/export_detection_calibration_evidence.sh "${BASE_URL}" "${summary_path}"
|
|
done < "$SAMPLE_REQUESTS"
|
|
|
|
"${PYTHON_BIN}" - "$PORTFOLIO_META" "$OUTPUT_DIR" "$CALIBRATION_EVIDENCE_MODE" <<'PY'
|
|
import json
|
|
import os
|
|
import sys
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
|
|
metadata_path, output_dir_raw, mode = sys.argv[1:4]
|
|
output_dir = Path(output_dir_raw)
|
|
metadata = json.loads(Path(metadata_path).read_text(encoding="utf-8"))
|
|
|
|
samples = []
|
|
total_features = 0
|
|
role_counts: dict[str, int] = {}
|
|
for sample in metadata["samples"]:
|
|
evidence_dir = Path(sample["evidence_dir"])
|
|
evidence_summary_path = evidence_dir / "calibration_evidence_summary.json"
|
|
evidence_geojson_path = evidence_dir / "calibration_evidence.geojson"
|
|
evidence_review_path = evidence_dir / "calibration_evidence_review.html"
|
|
if not evidence_summary_path.is_file():
|
|
raise SystemExit(f"Missing evidence summary for sample {sample['sample_slug']}: {evidence_summary_path}")
|
|
evidence_summary = json.loads(evidence_summary_path.read_text(encoding="utf-8"))
|
|
feature_count = int(evidence_summary.get("feature_count") or 0)
|
|
total_features += feature_count
|
|
for role, count in (evidence_summary.get("role_counts") or {}).items():
|
|
role_counts[role] = role_counts.get(role, 0) + int(count)
|
|
runs = evidence_summary.get("runs") or []
|
|
best_run = max(
|
|
[run for run in runs if run.get("quality_score") is not None],
|
|
key=lambda run: run["quality_score"],
|
|
default=None,
|
|
)
|
|
samples.append(
|
|
{
|
|
"sample_slug": sample["sample_slug"],
|
|
"aoi_label": sample["aoi_label"],
|
|
"operator_notes": sample["operator_notes"],
|
|
"source_summary_path": sample["source_summary_path"],
|
|
"copied_summary_path": sample["copied_summary_path"],
|
|
"evidence_summary_path": str(evidence_summary_path),
|
|
"evidence_geojson_path": str(evidence_geojson_path),
|
|
"evidence_review_path": str(evidence_review_path),
|
|
"evidence_feature_count": feature_count,
|
|
"role_counts": evidence_summary.get("role_counts") or {},
|
|
"best_run_by_score": best_run,
|
|
"runs": runs,
|
|
}
|
|
)
|
|
|
|
|
|
def sample_score(item: dict) -> float:
|
|
best = item.get("best_run_by_score") or {}
|
|
value = best.get("quality_score")
|
|
return value if isinstance(value, (int, float)) else float("-inf")
|
|
|
|
|
|
best_sample = max(samples, key=sample_score, default=None)
|
|
portfolio = {
|
|
"generated_at": datetime.now(timezone.utc).isoformat(),
|
|
"portfolio_name": metadata["portfolio_name"],
|
|
"model_asset_id": metadata.get("model_asset_id"),
|
|
"model_sha256": metadata.get("model_sha256"),
|
|
"notes": metadata.get("notes") or "",
|
|
"mode": mode,
|
|
"sample_count": len(samples),
|
|
"total_evidence_feature_count": total_features,
|
|
"role_counts": dict(sorted(role_counts.items())),
|
|
"best_sample_by_score": best_sample,
|
|
"samples": samples,
|
|
}
|
|
|
|
portfolio_path = output_dir / "calibration_evidence_portfolio.json"
|
|
portfolio_path.write_text(json.dumps(portfolio, indent=2, sort_keys=True), encoding="utf-8")
|
|
|
|
markdown_path = output_dir / "calibration_evidence_portfolio.md"
|
|
lines = [
|
|
f"# {portfolio['portfolio_name']}",
|
|
"",
|
|
f"- Generated: {portfolio['generated_at']}",
|
|
f"- Model asset: {portfolio.get('model_asset_id') or 'not recorded'}",
|
|
f"- Model SHA256: {portfolio.get('model_sha256') or 'not recorded'}",
|
|
f"- Mode: {portfolio['mode']}",
|
|
f"- Samples: {portfolio['sample_count']}",
|
|
f"- Evidence features: {portfolio['total_evidence_feature_count']}",
|
|
]
|
|
if portfolio.get("notes"):
|
|
lines.extend(["", "## Operator notes", "", portfolio["notes"]])
|
|
lines.extend(["", "## Samples", ""])
|
|
for sample in samples:
|
|
best = sample.get("best_run_by_score") or {}
|
|
lines.extend(
|
|
[
|
|
f"### {sample['aoi_label']} (`{sample['sample_slug']}`)",
|
|
"",
|
|
f"- Evidence features: {sample['evidence_feature_count']}",
|
|
f"- Best threshold: {best.get('threshold')}",
|
|
f"- Score: {best.get('quality_score')}",
|
|
f"- Precision: {best.get('precision')}",
|
|
f"- Recall: {best.get('recall')}",
|
|
f"- F1: {best.get('f1_score')}",
|
|
f"- Review HTML: `{sample['evidence_review_path']}`",
|
|
f"- Evidence GeoJSON: `{sample['evidence_geojson_path']}`",
|
|
f"- Evidence summary: `{sample['evidence_summary_path']}`",
|
|
]
|
|
)
|
|
if sample.get("operator_notes"):
|
|
lines.append(f"- Notes: {sample['operator_notes']}")
|
|
lines.append("")
|
|
markdown_path.write_text("\n".join(lines), encoding="utf-8")
|
|
|
|
print("Detection calibration evidence portfolio passed")
|
|
print(f"Portfolio JSON: {portfolio_path}")
|
|
print(f"Portfolio Markdown: {markdown_path}")
|
|
print(f"Samples: {portfolio['sample_count']}")
|
|
print(f"Evidence features: {portfolio['total_evidence_feature_count']}")
|
|
PY
|