Initial public release

scripts/run_detection_calibration_sweep.sh

@@ -0,0 +1,249 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+usage() {
+  cat >&2 <<'EOF'
+Usage:
+  REAL_RASTER_PATH=/path/to/orthophoto.tif \
+  REAL_REFERENCE_VECTOR_PATH=/path/to/reference-buildings.geojson \
+  CALIBRATION_THRESHOLDS="0.50 0.35 0.25 0.15" \
+  bash scripts/run_detection_calibration_sweep.sh [base_url]
+
+or:
+  bash scripts/run_detection_calibration_sweep.sh [base_url] /path/to/orthophoto.tif /path/to/reference-buildings.geojson
+
+Required inputs:
+  REAL_RASTER_PATH             Georeferenced .tif/.tiff/.geotiff raster.
+  REAL_REFERENCE_VECTOR_PATH   EPSG-aware .geojson/.json reference vector with building polygons.
+
+Optional environment:
+  CALIBRATION_THRESHOLDS       Space/comma separated confidence thresholds, default: 0.50 0.35 0.25 0.15.
+  CALIBRATION_OUTPUT_DIR       Output directory, default: artifacts/detection-calibration/<timestamp>.
+  REAL_MODEL_ASSET_ID          Specific /api/v1/detection/model-assets id to use.
+  REAL_TILE_SIZE               Raster tile size passed to the underlying real-data workflow.
+  REAL_TILE_OVERLAP            Raster tile overlap passed to the underlying real-data workflow.
+  REAL_IOU_THRESHOLD           QA IoU threshold, default inherited by the underlying workflow.
+EOF
+}
+
+ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+cd "$ROOT"
+
+BASE_URL="${1:-${GE_INTEL_BASE_URL:-http://localhost:1202}}"
+REAL_RASTER_PATH="${2:-${REAL_RASTER_PATH:-}}"
+REAL_REFERENCE_VECTOR_PATH="${3:-${REAL_REFERENCE_VECTOR_PATH:-}}"
+CALIBRATION_THRESHOLDS="${CALIBRATION_THRESHOLDS:-0.50 0.35 0.25 0.15}"
+CALIBRATION_OUTPUT_DIR="${CALIBRATION_OUTPUT_DIR:-artifacts/detection-calibration/$(date -u +%Y%m%dT%H%M%SZ)}"
+
+if [ "${BASE_URL}" = "-h" ] || [ "${BASE_URL}" = "--help" ]; then
+  usage
+  exit 0
+fi
+
+if [ -z "${REAL_RASTER_PATH}" ] || [ -z "${REAL_REFERENCE_VECTOR_PATH}" ]; then
+  usage
+  exit 2
+fi
+
+if ! command -v curl >/dev/null 2>&1; then
+  echo "curl is required for detection calibration sweep verification" >&2
+  exit 1
+fi
+
+if [ -n "${PYTHON_BIN:-}" ]; then
+  PYTHON_BIN="${PYTHON_BIN}"
+else
+  PYTHON_BIN=""
+  for candidate in python3 python.exe python; do
+    if command -v "${candidate}" >/dev/null 2>&1 && "${candidate}" -c "import json, sys" >/dev/null 2>&1; then
+      PYTHON_BIN="${candidate}"
+      break
+    fi
+  done
+fi
+
+if [ -z "${PYTHON_BIN}" ]; then
+  echo "A Python interpreter is required for JSON parsing" >&2
+  exit 1
+fi
+
+thresholds_normalized="$(printf '%s' "${CALIBRATION_THRESHOLDS}" | tr ',' ' ')"
+mkdir -p "${CALIBRATION_OUTPUT_DIR}"
+
+"${PYTHON_BIN}" - "${thresholds_normalized}" <<'PY'
+import sys
+
+raw = sys.argv[1].split()
+if not raw:
+    raise SystemExit("CALIBRATION_THRESHOLDS must contain at least one threshold")
+for value in raw:
+    try:
+        threshold = float(value)
+    except ValueError as exc:
+        raise SystemExit(f"Invalid confidence threshold: {value}") from exc
+    if threshold < 0.0 or threshold > 1.0:
+        raise SystemExit(f"Confidence threshold must be between 0 and 1: {value}")
+PY
+
+echo "== GeoIntel detection calibration sweep =="
+echo "Base URL: ${BASE_URL}"
+echo "Raster: ${REAL_RASTER_PATH}"
+echo "Reference vector: ${REAL_REFERENCE_VECTOR_PATH}"
+echo "Thresholds: ${thresholds_normalized}"
+echo "Output: ${CALIBRATION_OUTPUT_DIR}"
+
+run_index=0
+for threshold in ${thresholds_normalized}; do
+  run_index=$((run_index + 1))
+  threshold_label="$(printf '%s' "${threshold}" | tr '.-' 'pm')"
+  run_log="${CALIBRATION_OUTPUT_DIR}/threshold_${threshold_label}.log"
+  quality_response="${CALIBRATION_OUTPUT_DIR}/threshold_${threshold_label}_quality_checks.json"
+  detection_run_response="${CALIBRATION_OUTPUT_DIR}/threshold_${threshold_label}_detection_run.json"
+  run_summary="${CALIBRATION_OUTPUT_DIR}/threshold_${threshold_label}_summary.json"
+
+  echo "-- Threshold ${threshold} (${run_index}) --"
+  if ! REAL_PROJECT_NAME="GeoIntel Detection Calibration ${threshold}" \
+      REAL_CONFIDENCE_THRESHOLD="${threshold}" \
+      bash scripts/verify_real_data_detection_qa_workflow.sh "${BASE_URL}" "${REAL_RASTER_PATH}" "${REAL_REFERENCE_VECTOR_PATH}" \
+      >"${run_log}" 2>&1; then
+    echo "Calibration threshold ${threshold} failed. Log: ${run_log}" >&2
+    tail -n 80 "${run_log}" >&2 || true
+    exit 1
+  fi
+
+  project_id="$(sed -n 's/^Project: //p' "${run_log}" | tail -n 1)"
+  analysis_run_id="$(sed -n 's/^Analysis run: //p' "${run_log}" | tail -n 1)"
+  quality_check_id="$(sed -n 's/^Quality check: //p' "${run_log}" | tail -n 1)"
+  detection_count="$(sed -n 's/^Detections: //p' "${run_log}" | tail -n 1)"
+
+  if [ -z "${project_id}" ] || [ -z "${analysis_run_id}" ] || [ -z "${quality_check_id}" ]; then
+    echo "Calibration threshold ${threshold} did not print project/run/quality ids. Log: ${run_log}" >&2
+    exit 1
+  fi
+
+  curl -fsS "${BASE_URL%/}/api/v1/projects/${project_id}/quality-checks?limit=200" > "${quality_response}"
+  curl -fsS "${BASE_URL%/}/api/v1/detection/runs/${analysis_run_id}" > "${detection_run_response}"
+
+  "${PYTHON_BIN}" - \
+    "${quality_response}" \
+    "${detection_run_response}" \
+    "${run_summary}" \
+    "${threshold}" \
+    "${project_id}" \
+    "${analysis_run_id}" \
+    "${quality_check_id}" \
+    "${detection_count}" \
+    "${run_log}" <<'PY'
+import json
+import sys
+
+quality_path, detection_run_path, output_path, threshold, project_id, analysis_run_id, quality_check_id, detection_count, run_log = sys.argv[1:10]
+with open(quality_path, "r", encoding="utf-8") as handle:
+    payload = json.load(handle)
+if "data" not in payload:
+    raise SystemExit("Quality-check list is not a canonical GeoIntel data envelope")
+with open(detection_run_path, "r", encoding="utf-8") as handle:
+    detection_run_payload = json.load(handle)
+if "data" not in detection_run_payload:
+    raise SystemExit("Detection run detail is not a canonical GeoIntel data envelope")
+detection_run = detection_run_payload["data"]
+run_result = detection_run.get("result_json") or {}
+items = payload["data"].get("items") or []
+quality_check = next((item for item in items if str(item.get("id")) == quality_check_id), None)
+if quality_check is None:
+    raise SystemExit(f"Quality check not found in project list: {quality_check_id}")
+metrics = {
+    metric.get("metric_key"): metric.get("metric_value")
+    for metric in quality_check.get("metrics", [])
+    if metric.get("metric_key")
+}
+findings = quality_check.get("findings_json") or {}
+summary = {
+    "threshold": float(threshold),
+    "project_id": project_id,
+    "analysis_run_id": analysis_run_id,
+    "quality_check_id": quality_check_id,
+    "detection_count": int(detection_count),
+    "quality_status": quality_check.get("status"),
+    "quality_score": quality_check.get("score"),
+    "raw_detection_count": run_result.get("raw_detection_count", int(detection_count)),
+    "suppressed_detection_count": run_result.get("suppressed_detection_count", 0),
+    "duplicate_iou_threshold": run_result.get("duplicate_iou_threshold"),
+    "precision": metrics.get("precision"),
+    "recall": metrics.get("recall"),
+    "f1_score": metrics.get("f1_score", metrics.get("f1")),
+    "mean_iou": metrics.get("mean_iou"),
+    "matches": findings.get("matches"),
+    "false_positives": findings.get("false_positives"),
+    "false_negatives": findings.get("false_negatives"),
+    "run_log": run_log,
+}
+with open(output_path, "w", encoding="utf-8") as handle:
+    json.dump(summary, handle, indent=2, sort_keys=True)
+print(
+    "threshold={threshold} detections={detections} raw={raw} suppressed={suppressed} score={score} "
+    "precision={precision} recall={recall} f1={f1} matches={matches} fp={fp} fn={fn}".format(
+        threshold=summary["threshold"],
+        detections=summary["detection_count"],
+        raw=summary["raw_detection_count"],
+        suppressed=summary["suppressed_detection_count"],
+        score=summary["quality_score"],
+        precision=summary["precision"],
+        recall=summary["recall"],
+        f1=summary["f1_score"],
+        matches=summary["matches"],
+        fp=summary["false_positives"],
+        fn=summary["false_negatives"],
+    )
+)
+PY
+done
+
+"${PYTHON_BIN}" - "${CALIBRATION_OUTPUT_DIR}" "${BASE_URL}" "${thresholds_normalized}" <<'PY'
+import glob
+import json
+import os
+import sys
+from datetime import datetime, timezone
+
+output_dir, base_url, thresholds = sys.argv[1:4]
+items = []
+for path in sorted(glob.glob(os.path.join(output_dir, "threshold_*_summary.json"))):
+    with open(path, "r", encoding="utf-8") as handle:
+        items.append(json.load(handle))
+
+ranked = [
+    item
+    for item in items
+    if item.get("quality_score") is not None
+]
+best_by_score = max(ranked, key=lambda item: item["quality_score"], default=None)
+summary = {
+    "generated_at": datetime.now(timezone.utc).isoformat(),
+    "base_url": base_url,
+    "thresholds": [float(value) for value in thresholds.split()],
+    "best_by_score": best_by_score,
+    "items": items,
+}
+summary_path = os.path.join(output_dir, "calibration_summary.json")
+with open(summary_path, "w", encoding="utf-8") as handle:
+    json.dump(summary, handle, indent=2, sort_keys=True)
+
+print("")
+print("Detection calibration summary")
+print("threshold\tdetections\traw\tsuppressed\tscore\tprecision\trecall\tf1\tmatches\tfp\tfn")
+for item in items:
+    print(
+        "{threshold:.2f}\t{detection_count}\t{raw_detection_count}\t{suppressed_detection_count}\t{quality_score}\t{precision}\t{recall}\t{f1_score}\t{matches}\t{false_positives}\t{false_negatives}".format(
+            **item
+        )
+    )
+print("")
+print(f"Summary: {summary_path}")
+if best_by_score:
+    print(
+        "best_by_score threshold={threshold:.2f} score={quality_score} f1={f1_score} detections={detection_count}".format(
+            **best_by_score
+        )
+    )
+PY