462 lines
17 KiB
Bash
462 lines
17 KiB
Bash
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
|
|
usage() {
|
|
cat >&2 <<'EOF'
|
|
Usage:
|
|
CALIBRATION_SUMMARY_PATH=/path/to/calibration_summary.json \
|
|
bash scripts/export_detection_calibration_evidence.sh [base_url]
|
|
|
|
or:
|
|
bash scripts/export_detection_calibration_evidence.sh [base_url] /path/to/calibration_summary.json
|
|
|
|
or with the browser Detection Lab export:
|
|
bash scripts/export_detection_calibration_evidence.sh [base_url] /path/to/detection-calibration-summary.json
|
|
|
|
Required input:
|
|
CALIBRATION_SUMMARY_PATH calibration_summary.json produced by run_detection_calibration_sweep.sh
|
|
or detection-calibration-summary.json from the Detection Lab.
|
|
|
|
Optional environment:
|
|
CALIBRATION_EVIDENCE_MODE all or best, default: all.
|
|
CALIBRATION_EVIDENCE_DIR Output directory, default: the summary file directory.
|
|
CURL_BIN curl executable, default: curl.
|
|
EOF
|
|
}
|
|
|
|
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
|
cd "$ROOT"
|
|
|
|
BASE_URL="${1:-${GE_INTEL_BASE_URL:-http://localhost:1202}}"
|
|
CALIBRATION_SUMMARY_PATH="${2:-${CALIBRATION_SUMMARY_PATH:-}}"
|
|
CALIBRATION_EVIDENCE_MODE="${CALIBRATION_EVIDENCE_MODE:-all}"
|
|
|
|
if [ "${BASE_URL}" = "-h" ] || [ "${BASE_URL}" = "--help" ]; then
|
|
usage
|
|
exit 0
|
|
fi
|
|
|
|
if [ -z "${CALIBRATION_SUMMARY_PATH}" ]; then
|
|
usage
|
|
exit 2
|
|
fi
|
|
|
|
if [ ! -f "${CALIBRATION_SUMMARY_PATH}" ]; then
|
|
echo "CALIBRATION_SUMMARY_PATH does not point to a readable file: ${CALIBRATION_SUMMARY_PATH}" >&2
|
|
exit 2
|
|
fi
|
|
|
|
case "${CALIBRATION_EVIDENCE_MODE}" in
|
|
all|best) ;;
|
|
*)
|
|
echo "CALIBRATION_EVIDENCE_MODE must be 'all' or 'best'" >&2
|
|
exit 2
|
|
;;
|
|
esac
|
|
|
|
CURL_BIN="${CURL_BIN:-curl}"
|
|
if ! command -v "${CURL_BIN}" >/dev/null 2>&1; then
|
|
echo "curl is required for detection calibration evidence export" >&2
|
|
exit 1
|
|
fi
|
|
|
|
if [ -n "${PYTHON_BIN:-}" ]; then
|
|
PYTHON_BIN="${PYTHON_BIN}"
|
|
else
|
|
PYTHON_BIN=""
|
|
for candidate in python3 python.exe python; do
|
|
if command -v "${candidate}" >/dev/null 2>&1 && "${candidate}" -c "import json, sys" >/dev/null 2>&1; then
|
|
PYTHON_BIN="${candidate}"
|
|
break
|
|
fi
|
|
done
|
|
fi
|
|
|
|
if [ -z "${PYTHON_BIN}" ]; then
|
|
echo "A Python interpreter is required for JSON parsing" >&2
|
|
exit 1
|
|
fi
|
|
|
|
summary_dir="$(cd "$(dirname "${CALIBRATION_SUMMARY_PATH}")" && pwd)"
|
|
CALIBRATION_SUMMARY_PATH="$(cd "$(dirname "${CALIBRATION_SUMMARY_PATH}")" && pwd)/$(basename "${CALIBRATION_SUMMARY_PATH}")"
|
|
CALIBRATION_EVIDENCE_DIR="${CALIBRATION_EVIDENCE_DIR:-${summary_dir}}"
|
|
mkdir -p "${CALIBRATION_EVIDENCE_DIR}"
|
|
CALIBRATION_EVIDENCE_DIR="$(cd "${CALIBRATION_EVIDENCE_DIR}" && pwd)"
|
|
|
|
request_manifest="${CALIBRATION_EVIDENCE_DIR}/calibration_evidence_requests.tsv"
|
|
"${PYTHON_BIN}" - "${CALIBRATION_SUMMARY_PATH}" "${request_manifest}" "${CALIBRATION_EVIDENCE_MODE}" <<'PY'
|
|
import json
|
|
import sys
|
|
|
|
summary_path, manifest_path, mode = sys.argv[1:4]
|
|
with open(summary_path, "r", encoding="utf-8") as handle:
|
|
summary = json.load(handle)
|
|
|
|
def normalize_calibration_items(summary):
|
|
root_project_id = summary.get("project_id")
|
|
if summary.get("export_type") == "detection_calibration_summary":
|
|
rows = summary.get("rows") or []
|
|
quality_check_ids = summary.get("quality_check_ids") or []
|
|
items = []
|
|
for row in rows:
|
|
if not isinstance(row, dict):
|
|
continue
|
|
quality_check_id = row.get("quality_check_id")
|
|
if not quality_check_id:
|
|
continue
|
|
items.append(
|
|
{
|
|
"project_id": row.get("project_id") or root_project_id,
|
|
"analysis_run_id": row.get("analysis_run_id"),
|
|
"job_id": row.get("job_id"),
|
|
"quality_check_id": quality_check_id,
|
|
"threshold": row.get("threshold"),
|
|
"detection_count": row.get("detection_count"),
|
|
"quality_score": row.get("quality_score") or row.get("f1_score"),
|
|
"precision": row.get("precision"),
|
|
"recall": row.get("recall"),
|
|
"f1_score": row.get("f1_score"),
|
|
}
|
|
)
|
|
if quality_check_ids and not items:
|
|
raise SystemExit("detection-calibration-summary.json has quality_check_ids but no export rows")
|
|
return items
|
|
return summary.get("items") or []
|
|
|
|
items = normalize_calibration_items(summary)
|
|
if mode == "best":
|
|
best = summary.get("best_by_score")
|
|
if best is None and items:
|
|
best = max(items, key=lambda item: item.get("quality_score") if isinstance(item.get("quality_score"), (int, float)) else float("-inf"))
|
|
if not isinstance(best, dict):
|
|
raise SystemExit("calibration_summary.json has no best_by_score object")
|
|
items = [best]
|
|
|
|
if not items:
|
|
raise SystemExit("calibration_summary.json contains no calibration items")
|
|
|
|
with open(manifest_path, "w", encoding="utf-8") as handle:
|
|
for item in items:
|
|
project_id = item.get("project_id")
|
|
quality_check_id = item.get("quality_check_id")
|
|
threshold = item.get("threshold")
|
|
if not project_id or not quality_check_id:
|
|
raise SystemExit("Calibration item is missing project_id or quality_check_id")
|
|
handle.write(f"{threshold}\t{project_id}\t{quality_check_id}\n")
|
|
PY
|
|
|
|
echo "== GeoIntel detection calibration evidence export =="
|
|
echo "Base URL: ${BASE_URL}"
|
|
echo "Summary: ${CALIBRATION_SUMMARY_PATH}"
|
|
echo "Mode: ${CALIBRATION_EVIDENCE_MODE}"
|
|
echo "Output: ${CALIBRATION_EVIDENCE_DIR}"
|
|
|
|
while IFS=$'\t' read -r threshold project_id quality_check_id; do
|
|
threshold_label="$(printf '%s' "${threshold}" | tr '.-' 'pm')"
|
|
response_path="${CALIBRATION_EVIDENCE_DIR}/threshold_${threshold_label}_evidence_response.json"
|
|
echo "-- Evidence threshold ${threshold}, quality_check_id ${quality_check_id} --"
|
|
"${CURL_BIN}" -fsS "${BASE_URL%/}/api/v1/projects/${project_id}/quality-checks/${quality_check_id}/evidence/geojson" > "${response_path}"
|
|
done < "${request_manifest}"
|
|
|
|
"${PYTHON_BIN}" - "${CALIBRATION_SUMMARY_PATH}" "${CALIBRATION_EVIDENCE_DIR}" "${CALIBRATION_EVIDENCE_MODE}" <<'PY'
|
|
import glob
|
|
import html
|
|
import json
|
|
import math
|
|
import os
|
|
import sys
|
|
from collections import Counter
|
|
from datetime import datetime, timezone
|
|
|
|
summary_path, output_dir, mode = sys.argv[1:4]
|
|
with open(summary_path, "r", encoding="utf-8") as handle:
|
|
calibration_summary = json.load(handle)
|
|
|
|
def normalize_calibration_items(summary):
|
|
root_project_id = summary.get("project_id")
|
|
if summary.get("export_type") == "detection_calibration_summary":
|
|
rows = summary.get("rows") or []
|
|
quality_check_ids = summary.get("quality_check_ids") or []
|
|
items = []
|
|
for row in rows:
|
|
if not isinstance(row, dict):
|
|
continue
|
|
quality_check_id = row.get("quality_check_id")
|
|
if not quality_check_id:
|
|
continue
|
|
items.append(
|
|
{
|
|
"project_id": row.get("project_id") or root_project_id,
|
|
"analysis_run_id": row.get("analysis_run_id"),
|
|
"job_id": row.get("job_id"),
|
|
"quality_check_id": quality_check_id,
|
|
"threshold": row.get("threshold"),
|
|
"detection_count": row.get("detection_count"),
|
|
"quality_score": row.get("quality_score") or row.get("f1_score"),
|
|
"precision": row.get("precision"),
|
|
"recall": row.get("recall"),
|
|
"f1_score": row.get("f1_score"),
|
|
}
|
|
)
|
|
if quality_check_ids and not items:
|
|
raise SystemExit("detection-calibration-summary.json has quality_check_ids but no export rows")
|
|
return items
|
|
return summary.get("items") or []
|
|
|
|
selected_items = normalize_calibration_items(calibration_summary)
|
|
if mode == "best":
|
|
best = calibration_summary.get("best_by_score")
|
|
if best is None and selected_items:
|
|
best = max(
|
|
selected_items,
|
|
key=lambda item: item.get("quality_score") if isinstance(item.get("quality_score"), (int, float)) else float("-inf"),
|
|
)
|
|
selected_items = [best]
|
|
items_by_quality_check = {str(item["quality_check_id"]): item for item in selected_items}
|
|
|
|
combined_features = []
|
|
evidence_runs = []
|
|
role_counts = Counter()
|
|
warnings = []
|
|
|
|
for response_path in sorted(glob.glob(os.path.join(output_dir, "threshold_*_evidence_response.json"))):
|
|
with open(response_path, "r", encoding="utf-8") as handle:
|
|
payload = json.load(handle)
|
|
if "data" not in payload:
|
|
raise SystemExit("Response is not a canonical GeoIntel data envelope")
|
|
data = payload["data"]
|
|
quality_check_id = str(data.get("quality_check_id"))
|
|
item = items_by_quality_check.get(quality_check_id)
|
|
if item is None:
|
|
continue
|
|
threshold = item.get("threshold")
|
|
run_warnings = data.get("warnings") or []
|
|
warnings.extend(run_warnings)
|
|
feature_count_before = len(combined_features)
|
|
for feature in (data.get("geojson") or {}).get("features") or []:
|
|
properties = dict(feature.get("properties") or {})
|
|
role = properties.get("qa_evidence_role") or "unknown"
|
|
role_counts[role] += 1
|
|
properties.update(
|
|
{
|
|
"calibration_threshold": threshold,
|
|
"calibration_quality_score": item.get("quality_score"),
|
|
"calibration_precision": item.get("precision"),
|
|
"calibration_recall": item.get("recall"),
|
|
"calibration_f1_score": item.get("f1_score"),
|
|
"calibration_detection_count": item.get("detection_count"),
|
|
}
|
|
)
|
|
enriched = dict(feature)
|
|
enriched["properties"] = properties
|
|
enriched["id"] = f"{threshold}:{feature.get('id')}"
|
|
combined_features.append(enriched)
|
|
evidence_runs.append(
|
|
{
|
|
"threshold": threshold,
|
|
"project_id": item.get("project_id"),
|
|
"analysis_run_id": item.get("analysis_run_id"),
|
|
"quality_check_id": quality_check_id,
|
|
"detection_count": item.get("detection_count"),
|
|
"quality_score": item.get("quality_score"),
|
|
"precision": item.get("precision"),
|
|
"recall": item.get("recall"),
|
|
"f1_score": item.get("f1_score"),
|
|
"evidence_feature_count": len(combined_features) - feature_count_before,
|
|
"warnings": run_warnings,
|
|
}
|
|
)
|
|
|
|
combined_geojson = {
|
|
"type": "FeatureCollection",
|
|
"features": combined_features,
|
|
}
|
|
combined_path = os.path.join(output_dir, "calibration_evidence.geojson")
|
|
with open(combined_path, "w", encoding="utf-8") as handle:
|
|
json.dump(combined_geojson, handle, indent=2, sort_keys=True)
|
|
|
|
summary = {
|
|
"generated_at": datetime.now(timezone.utc).isoformat(),
|
|
"mode": mode,
|
|
"source_summary_path": summary_path,
|
|
"feature_count": len(combined_features),
|
|
"role_counts": dict(sorted(role_counts.items())),
|
|
"warnings": warnings,
|
|
"runs": evidence_runs,
|
|
"geojson_path": combined_path,
|
|
}
|
|
bundle_summary_path = os.path.join(output_dir, "calibration_evidence_summary.json")
|
|
with open(bundle_summary_path, "w", encoding="utf-8") as handle:
|
|
json.dump(summary, handle, indent=2, sort_keys=True)
|
|
|
|
|
|
def iter_coords(geometry):
|
|
if not isinstance(geometry, dict):
|
|
return
|
|
coords = geometry.get("coordinates")
|
|
if geometry.get("type") == "Point" and isinstance(coords, list) and len(coords) >= 2:
|
|
yield float(coords[0]), float(coords[1])
|
|
elif isinstance(coords, list):
|
|
stack = [coords]
|
|
while stack:
|
|
item = stack.pop()
|
|
if (
|
|
isinstance(item, list)
|
|
and len(item) >= 2
|
|
and all(isinstance(value, (int, float)) for value in item[:2])
|
|
):
|
|
yield float(item[0]), float(item[1])
|
|
elif isinstance(item, list):
|
|
stack.extend(item)
|
|
|
|
|
|
coordinates = [coord for feature in combined_features for coord in iter_coords(feature.get("geometry"))]
|
|
if coordinates:
|
|
min_x = min(x for x, _ in coordinates)
|
|
max_x = max(x for x, _ in coordinates)
|
|
min_y = min(y for _, y in coordinates)
|
|
max_y = max(y for _, y in coordinates)
|
|
else:
|
|
min_x = min_y = 0.0
|
|
max_x = max_y = 1.0
|
|
width = max(max_x - min_x, 0.000001)
|
|
height = max(max_y - min_y, 0.000001)
|
|
|
|
|
|
def project(x, y):
|
|
px = ((x - min_x) / width) * 1120 + 40
|
|
py = 760 - (((y - min_y) / height) * 720 + 20)
|
|
return px, py
|
|
|
|
|
|
def path_for_geometry(geometry):
|
|
projected = [project(x, y) for x, y in iter_coords(geometry)]
|
|
if not projected:
|
|
return ""
|
|
return "M " + " L ".join(f"{x:.2f} {y:.2f}" for x, y in projected) + " Z"
|
|
|
|
|
|
role_colors = {
|
|
"match_candidate": "#0f766e",
|
|
"match_reference": "#22c55e",
|
|
"false_positive": "#dc2626",
|
|
"false_negative": "#2563eb",
|
|
}
|
|
paths = []
|
|
for feature in combined_features:
|
|
properties = feature.get("properties") or {}
|
|
role = properties.get("qa_evidence_role", "unknown")
|
|
threshold = properties.get("calibration_threshold")
|
|
stroke = role_colors.get(role, "#64748b")
|
|
path_data = path_for_geometry(feature.get("geometry"))
|
|
if not path_data:
|
|
continue
|
|
title = html.escape(f"threshold={threshold} role={role} feature={properties.get('feature_id')}")
|
|
paths.append(
|
|
f'<path d="{path_data}" fill="none" stroke="{stroke}" stroke-width="1.5" opacity="0.72">'
|
|
f"<title>{title}</title></path>"
|
|
)
|
|
|
|
rows = []
|
|
for run in sorted(evidence_runs, key=lambda item: float(item.get("threshold") or 0)):
|
|
rows.append(
|
|
"<tr>"
|
|
f"<td>{run.get('threshold')}</td>"
|
|
f"<td>{run.get('detection_count')}</td>"
|
|
f"<td>{run.get('quality_score')}</td>"
|
|
f"<td>{run.get('precision')}</td>"
|
|
f"<td>{run.get('recall')}</td>"
|
|
f"<td>{run.get('f1_score')}</td>"
|
|
f"<td>{run.get('evidence_feature_count')}</td>"
|
|
f"<td>{html.escape(str(run.get('quality_check_id')))}</td>"
|
|
"</tr>"
|
|
)
|
|
|
|
role_items = "".join(
|
|
f"<li><span>{html.escape(role)}</span><strong>{count}</strong></li>"
|
|
for role, count in sorted(role_counts.items())
|
|
)
|
|
|
|
html_path = os.path.join(output_dir, "calibration_evidence_review.html")
|
|
with open(html_path, "w", encoding="utf-8") as handle:
|
|
handle.write(
|
|
"""<!doctype html>
|
|
<html lang="en">
|
|
<head>
|
|
<meta charset="utf-8">
|
|
<meta name="viewport" content="width=device-width, initial-scale=1">
|
|
<title>GeoIntel Detection Calibration Evidence</title>
|
|
<style>
|
|
:root { color-scheme: light; font-family: Inter, Arial, sans-serif; background: #f6f7f3; color: #16201b; }
|
|
body { margin: 0; padding: 24px; }
|
|
main { max-width: 1240px; margin: 0 auto; }
|
|
h1 { font-size: 28px; margin: 0 0 8px; }
|
|
p { color: #526158; }
|
|
.grid { display: grid; grid-template-columns: minmax(0, 1fr) 280px; gap: 16px; align-items: start; }
|
|
.panel { background: #fff; border: 1px solid #d8ded6; border-radius: 8px; padding: 16px; }
|
|
svg { width: 100%; height: auto; background: #f8faf7; border: 1px solid #d8ded6; border-radius: 6px; }
|
|
table { width: 100%; border-collapse: collapse; font-size: 13px; }
|
|
th, td { text-align: left; border-bottom: 1px solid #e5e9e2; padding: 8px; vertical-align: top; }
|
|
ul { list-style: none; padding: 0; margin: 0; }
|
|
li { display: flex; justify-content: space-between; border-bottom: 1px solid #e5e9e2; padding: 8px 0; }
|
|
.legend span { display: inline-block; width: 11px; height: 11px; border-radius: 50%; margin-right: 8px; }
|
|
.match_candidate { background: #0f766e; }
|
|
.match_reference { background: #22c55e; }
|
|
.false_positive { background: #dc2626; }
|
|
.false_negative { background: #2563eb; }
|
|
@media (max-width: 860px) { .grid { grid-template-columns: 1fr; } body { padding: 12px; } }
|
|
</style>
|
|
</head>
|
|
<body>
|
|
<main>
|
|
<h1>Detection calibration evidence review</h1>
|
|
<p>Generated from persisted GeoIntel QualityCheck evidence. Coordinates are rendered directly from the evidence GeoJSON for quick inspection.</p>
|
|
<div class="grid">
|
|
<section class="panel">
|
|
<h2>Evidence geometry overview</h2>
|
|
<svg viewBox="0 0 1200 800" role="img" aria-label="QA evidence geometry overview">
|
|
"""
|
|
+ "\n".join(paths)
|
|
+ """
|
|
</svg>
|
|
</section>
|
|
<aside class="panel">
|
|
<h2>Role counts</h2>
|
|
<ul>
|
|
"""
|
|
+ role_items
|
|
+ """
|
|
</ul>
|
|
<h2>Legend</h2>
|
|
<p class="legend"><span class="match_candidate"></span>matched detection</p>
|
|
<p class="legend"><span class="match_reference"></span>matched reference</p>
|
|
<p class="legend"><span class="false_positive"></span>false positive</p>
|
|
<p class="legend"><span class="false_negative"></span>false negative</p>
|
|
</aside>
|
|
</div>
|
|
<section class="panel" style="margin-top: 16px;">
|
|
<h2>Calibration runs</h2>
|
|
<table>
|
|
<thead><tr><th>threshold</th><th>detections</th><th>score</th><th>precision</th><th>recall</th><th>f1</th><th>evidence</th><th>quality check</th></tr></thead>
|
|
<tbody>
|
|
"""
|
|
+ "\n".join(rows)
|
|
+ """
|
|
</tbody>
|
|
</table>
|
|
</section>
|
|
</main>
|
|
</body>
|
|
</html>
|
|
"""
|
|
)
|
|
|
|
print("Detection calibration evidence export passed")
|
|
print(f"Evidence GeoJSON: {combined_path}")
|
|
print(f"Evidence summary: {bundle_summary_path}")
|
|
print(f"Evidence review: {html_path}")
|
|
print(f"Evidence features: {len(combined_features)}")
|
|
for role, count in sorted(role_counts.items()):
|
|
print(f"{role}: {count}")
|
|
PY
|