#!/usr/bin/env bash set -euo pipefail usage() { cat >&2 <<'EOF' Usage: REAL_RASTER_PATH=/path/to/orthophoto.tif \ REAL_REFERENCE_VECTOR_PATH=/path/to/reference-buildings.geojson \ CALIBRATION_THRESHOLDS="0.50 0.35 0.25 0.15" \ bash scripts/run_detection_calibration_sweep.sh [base_url] or: bash scripts/run_detection_calibration_sweep.sh [base_url] /path/to/orthophoto.tif /path/to/reference-buildings.geojson Required inputs: REAL_RASTER_PATH Georeferenced .tif/.tiff/.geotiff raster. REAL_REFERENCE_VECTOR_PATH EPSG-aware .geojson/.json reference vector with building polygons. Optional environment: CALIBRATION_THRESHOLDS Space/comma separated confidence thresholds, default: 0.50 0.35 0.25 0.15. CALIBRATION_OUTPUT_DIR Output directory, default: artifacts/detection-calibration/. REAL_MODEL_ASSET_ID Specific /api/v1/detection/model-assets id to use. REAL_TILE_SIZE Raster tile size passed to the underlying real-data workflow. REAL_TILE_OVERLAP Raster tile overlap passed to the underlying real-data workflow. REAL_IOU_THRESHOLD QA IoU threshold, default inherited by the underlying workflow. EOF } ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" cd "$ROOT" BASE_URL="${1:-${GE_INTEL_BASE_URL:-http://localhost:1202}}" REAL_RASTER_PATH="${2:-${REAL_RASTER_PATH:-}}" REAL_REFERENCE_VECTOR_PATH="${3:-${REAL_REFERENCE_VECTOR_PATH:-}}" CALIBRATION_THRESHOLDS="${CALIBRATION_THRESHOLDS:-0.50 0.35 0.25 0.15}" CALIBRATION_OUTPUT_DIR="${CALIBRATION_OUTPUT_DIR:-artifacts/detection-calibration/$(date -u +%Y%m%dT%H%M%SZ)}" if [ "${BASE_URL}" = "-h" ] || [ "${BASE_URL}" = "--help" ]; then usage exit 0 fi if [ -z "${REAL_RASTER_PATH}" ] || [ -z "${REAL_REFERENCE_VECTOR_PATH}" ]; then usage exit 2 fi if ! command -v curl >/dev/null 2>&1; then echo "curl is required for detection calibration sweep verification" >&2 exit 1 fi if [ -n "${PYTHON_BIN:-}" ]; then PYTHON_BIN="${PYTHON_BIN}" else PYTHON_BIN="" for candidate in python3 python.exe python; do if command -v "${candidate}" >/dev/null 2>&1 && "${candidate}" -c "import json, sys" >/dev/null 2>&1; then PYTHON_BIN="${candidate}" break fi done fi if [ -z "${PYTHON_BIN}" ]; then echo "A Python interpreter is required for JSON parsing" >&2 exit 1 fi thresholds_normalized="$(printf '%s' "${CALIBRATION_THRESHOLDS}" | tr ',' ' ')" mkdir -p "${CALIBRATION_OUTPUT_DIR}" "${PYTHON_BIN}" - "${thresholds_normalized}" <<'PY' import sys raw = sys.argv[1].split() if not raw: raise SystemExit("CALIBRATION_THRESHOLDS must contain at least one threshold") for value in raw: try: threshold = float(value) except ValueError as exc: raise SystemExit(f"Invalid confidence threshold: {value}") from exc if threshold < 0.0 or threshold > 1.0: raise SystemExit(f"Confidence threshold must be between 0 and 1: {value}") PY echo "== GeoIntel detection calibration sweep ==" echo "Base URL: ${BASE_URL}" echo "Raster: ${REAL_RASTER_PATH}" echo "Reference vector: ${REAL_REFERENCE_VECTOR_PATH}" echo "Thresholds: ${thresholds_normalized}" echo "Output: ${CALIBRATION_OUTPUT_DIR}" run_index=0 for threshold in ${thresholds_normalized}; do run_index=$((run_index + 1)) threshold_label="$(printf '%s' "${threshold}" | tr '.-' 'pm')" run_log="${CALIBRATION_OUTPUT_DIR}/threshold_${threshold_label}.log" quality_response="${CALIBRATION_OUTPUT_DIR}/threshold_${threshold_label}_quality_checks.json" detection_run_response="${CALIBRATION_OUTPUT_DIR}/threshold_${threshold_label}_detection_run.json" run_summary="${CALIBRATION_OUTPUT_DIR}/threshold_${threshold_label}_summary.json" echo "-- Threshold ${threshold} (${run_index}) --" if ! REAL_PROJECT_NAME="GeoIntel Detection Calibration ${threshold}" \ REAL_CONFIDENCE_THRESHOLD="${threshold}" \ bash scripts/verify_real_data_detection_qa_workflow.sh "${BASE_URL}" "${REAL_RASTER_PATH}" "${REAL_REFERENCE_VECTOR_PATH}" \ >"${run_log}" 2>&1; then echo "Calibration threshold ${threshold} failed. Log: ${run_log}" >&2 tail -n 80 "${run_log}" >&2 || true exit 1 fi project_id="$(sed -n 's/^Project: //p' "${run_log}" | tail -n 1)" analysis_run_id="$(sed -n 's/^Analysis run: //p' "${run_log}" | tail -n 1)" quality_check_id="$(sed -n 's/^Quality check: //p' "${run_log}" | tail -n 1)" detection_count="$(sed -n 's/^Detections: //p' "${run_log}" | tail -n 1)" if [ -z "${project_id}" ] || [ -z "${analysis_run_id}" ] || [ -z "${quality_check_id}" ]; then echo "Calibration threshold ${threshold} did not print project/run/quality ids. Log: ${run_log}" >&2 exit 1 fi curl -fsS "${BASE_URL%/}/api/v1/projects/${project_id}/quality-checks?limit=200" > "${quality_response}" curl -fsS "${BASE_URL%/}/api/v1/detection/runs/${analysis_run_id}" > "${detection_run_response}" "${PYTHON_BIN}" - \ "${quality_response}" \ "${detection_run_response}" \ "${run_summary}" \ "${threshold}" \ "${project_id}" \ "${analysis_run_id}" \ "${quality_check_id}" \ "${detection_count}" \ "${run_log}" <<'PY' import json import sys quality_path, detection_run_path, output_path, threshold, project_id, analysis_run_id, quality_check_id, detection_count, run_log = sys.argv[1:10] with open(quality_path, "r", encoding="utf-8") as handle: payload = json.load(handle) if "data" not in payload: raise SystemExit("Quality-check list is not a canonical GeoIntel data envelope") with open(detection_run_path, "r", encoding="utf-8") as handle: detection_run_payload = json.load(handle) if "data" not in detection_run_payload: raise SystemExit("Detection run detail is not a canonical GeoIntel data envelope") detection_run = detection_run_payload["data"] run_result = detection_run.get("result_json") or {} items = payload["data"].get("items") or [] quality_check = next((item for item in items if str(item.get("id")) == quality_check_id), None) if quality_check is None: raise SystemExit(f"Quality check not found in project list: {quality_check_id}") metrics = { metric.get("metric_key"): metric.get("metric_value") for metric in quality_check.get("metrics", []) if metric.get("metric_key") } findings = quality_check.get("findings_json") or {} summary = { "threshold": float(threshold), "project_id": project_id, "analysis_run_id": analysis_run_id, "quality_check_id": quality_check_id, "detection_count": int(detection_count), "quality_status": quality_check.get("status"), "quality_score": quality_check.get("score"), "raw_detection_count": run_result.get("raw_detection_count", int(detection_count)), "suppressed_detection_count": run_result.get("suppressed_detection_count", 0), "duplicate_iou_threshold": run_result.get("duplicate_iou_threshold"), "precision": metrics.get("precision"), "recall": metrics.get("recall"), "f1_score": metrics.get("f1_score", metrics.get("f1")), "mean_iou": metrics.get("mean_iou"), "matches": findings.get("matches"), "false_positives": findings.get("false_positives"), "false_negatives": findings.get("false_negatives"), "run_log": run_log, } with open(output_path, "w", encoding="utf-8") as handle: json.dump(summary, handle, indent=2, sort_keys=True) print( "threshold={threshold} detections={detections} raw={raw} suppressed={suppressed} score={score} " "precision={precision} recall={recall} f1={f1} matches={matches} fp={fp} fn={fn}".format( threshold=summary["threshold"], detections=summary["detection_count"], raw=summary["raw_detection_count"], suppressed=summary["suppressed_detection_count"], score=summary["quality_score"], precision=summary["precision"], recall=summary["recall"], f1=summary["f1_score"], matches=summary["matches"], fp=summary["false_positives"], fn=summary["false_negatives"], ) ) PY done "${PYTHON_BIN}" - "${CALIBRATION_OUTPUT_DIR}" "${BASE_URL}" "${thresholds_normalized}" <<'PY' import glob import json import os import sys from datetime import datetime, timezone output_dir, base_url, thresholds = sys.argv[1:4] items = [] for path in sorted(glob.glob(os.path.join(output_dir, "threshold_*_summary.json"))): with open(path, "r", encoding="utf-8") as handle: items.append(json.load(handle)) ranked = [ item for item in items if item.get("quality_score") is not None ] best_by_score = max(ranked, key=lambda item: item["quality_score"], default=None) summary = { "generated_at": datetime.now(timezone.utc).isoformat(), "base_url": base_url, "thresholds": [float(value) for value in thresholds.split()], "best_by_score": best_by_score, "items": items, } summary_path = os.path.join(output_dir, "calibration_summary.json") with open(summary_path, "w", encoding="utf-8") as handle: json.dump(summary, handle, indent=2, sort_keys=True) print("") print("Detection calibration summary") print("threshold\tdetections\traw\tsuppressed\tscore\tprecision\trecall\tf1\tmatches\tfp\tfn") for item in items: print( "{threshold:.2f}\t{detection_count}\t{raw_detection_count}\t{suppressed_detection_count}\t{quality_score}\t{precision}\t{recall}\t{f1_score}\t{matches}\t{false_positives}\t{false_negatives}".format( **item ) ) print("") print(f"Summary: {summary_path}") if best_by_score: print( "best_by_score threshold={threshold:.2f} score={quality_score} f1={f1_score} detections={detection_count}".format( **best_by_score ) ) PY