392 lines
15 KiB
Bash
392 lines
15 KiB
Bash
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
|
|
usage() {
|
|
cat >&2 <<'EOF'
|
|
Usage:
|
|
REAL_RASTER_PATH=/path/to/orthophoto.tif \
|
|
REAL_REFERENCE_VECTOR_PATH=/path/to/reference-buildings.geojson \
|
|
QUALITY_MODEL_ASSET_IDS="yolov8n-building-segmentation-pt yolov8n-pt" \
|
|
QUALITY_TILE_SIZES="512 640" \
|
|
QUALITY_TILE_OVERLAPS="64" \
|
|
QUALITY_THRESHOLDS="0.50 0.25 0.15" \
|
|
bash scripts/run_detection_quality_matrix.sh [base_url]
|
|
|
|
or:
|
|
bash scripts/run_detection_quality_matrix.sh [base_url] /path/to/orthophoto.tif /path/to/reference-buildings.geojson
|
|
|
|
Required inputs:
|
|
REAL_RASTER_PATH Georeferenced .tif/.tiff/.geotiff raster.
|
|
REAL_REFERENCE_VECTOR_PATH EPSG-aware .geojson/.json reference vector with building polygons.
|
|
|
|
Optional environment:
|
|
QUALITY_MODEL_ASSET_IDS Space/comma separated local model asset IDs. Default: active configured model asset.
|
|
QUALITY_TILE_SIZES Space/comma separated raster tile sizes, default: 640.
|
|
QUALITY_TILE_OVERLAPS Space/comma separated raster tile overlaps, default: 64.
|
|
QUALITY_THRESHOLDS Space/comma separated confidence thresholds, default: 0.50 0.25 0.15.
|
|
QUALITY_OUTPUT_DIR Output directory, default: artifacts/detection-quality-matrix/<timestamp>.
|
|
REAL_IOU_THRESHOLD QA IoU threshold, default inherited by the underlying workflow.
|
|
|
|
This script never downloads models and never uses fixture detections. It repeats
|
|
the existing real-data upload/tile/detection/QA workflow for each matrix row.
|
|
EOF
|
|
}
|
|
|
|
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
|
cd "$ROOT"
|
|
|
|
BASE_URL="${1:-${GE_INTEL_BASE_URL:-http://localhost:1202}}"
|
|
REAL_RASTER_PATH="${2:-${REAL_RASTER_PATH:-}}"
|
|
REAL_REFERENCE_VECTOR_PATH="${3:-${REAL_REFERENCE_VECTOR_PATH:-}}"
|
|
QUALITY_MODEL_ASSET_IDS="${QUALITY_MODEL_ASSET_IDS:-${REAL_MODEL_ASSET_ID:-__active__}}"
|
|
QUALITY_TILE_SIZES="${QUALITY_TILE_SIZES:-640}"
|
|
QUALITY_TILE_OVERLAPS="${QUALITY_TILE_OVERLAPS:-64}"
|
|
QUALITY_THRESHOLDS="${QUALITY_THRESHOLDS:-0.50 0.25 0.15}"
|
|
QUALITY_OUTPUT_DIR="${QUALITY_OUTPUT_DIR:-artifacts/detection-quality-matrix/$(date -u +%Y%m%dT%H%M%SZ)}"
|
|
|
|
if [ "${BASE_URL}" = "-h" ] || [ "${BASE_URL}" = "--help" ]; then
|
|
usage
|
|
exit 0
|
|
fi
|
|
|
|
if [ -z "${REAL_RASTER_PATH}" ] || [ -z "${REAL_REFERENCE_VECTOR_PATH}" ]; then
|
|
usage
|
|
exit 2
|
|
fi
|
|
|
|
if ! command -v curl >/dev/null 2>&1; then
|
|
echo "curl is required for detection quality matrix verification" >&2
|
|
exit 1
|
|
fi
|
|
|
|
if [ -n "${PYTHON_BIN:-}" ]; then
|
|
PYTHON_BIN="${PYTHON_BIN}"
|
|
else
|
|
PYTHON_BIN=""
|
|
for candidate in python3 python.exe python; do
|
|
if command -v "${candidate}" >/dev/null 2>&1 && "${candidate}" -c "import json, sys" >/dev/null 2>&1; then
|
|
PYTHON_BIN="${candidate}"
|
|
break
|
|
fi
|
|
done
|
|
fi
|
|
|
|
if [ -z "${PYTHON_BIN}" ]; then
|
|
echo "A Python interpreter is required for JSON parsing" >&2
|
|
exit 1
|
|
fi
|
|
|
|
mkdir -p "${QUALITY_OUTPUT_DIR}"
|
|
|
|
model_requests_normalized="$(printf '%s' "${QUALITY_MODEL_ASSET_IDS}" | tr ',' ' ')"
|
|
tile_sizes_normalized="$(printf '%s' "${QUALITY_TILE_SIZES}" | tr ',' ' ')"
|
|
tile_overlaps_normalized="$(printf '%s' "${QUALITY_TILE_OVERLAPS}" | tr ',' ' ')"
|
|
thresholds_normalized="$(printf '%s' "${QUALITY_THRESHOLDS}" | tr ',' ' ')"
|
|
matrix_manifest="${QUALITY_OUTPUT_DIR}/quality_matrix_requests.tsv"
|
|
|
|
"${PYTHON_BIN}" - \
|
|
"${model_requests_normalized}" \
|
|
"${tile_sizes_normalized}" \
|
|
"${tile_overlaps_normalized}" \
|
|
"${thresholds_normalized}" \
|
|
"${matrix_manifest}" <<'PY'
|
|
import re
|
|
import sys
|
|
|
|
models, tile_sizes, tile_overlaps, thresholds, output_path = sys.argv[1:6]
|
|
|
|
|
|
def split_values(raw: str, name: str) -> list[str]:
|
|
values = [value.strip() for value in raw.split() if value.strip()]
|
|
if not values:
|
|
raise SystemExit(f"{name} must contain at least one value")
|
|
return values
|
|
|
|
|
|
def safe_label(value: str) -> str:
|
|
label = re.sub(r"[^A-Za-z0-9_.-]+", "_", value.strip())
|
|
return label.replace(".", "p").replace("-", "_") or "active"
|
|
|
|
|
|
model_values = split_values(models, "QUALITY_MODEL_ASSET_IDS")
|
|
tile_size_values = split_values(tile_sizes, "QUALITY_TILE_SIZES")
|
|
tile_overlap_values = split_values(tile_overlaps, "QUALITY_TILE_OVERLAPS")
|
|
threshold_values = split_values(thresholds, "QUALITY_THRESHOLDS")
|
|
|
|
for value in tile_size_values:
|
|
try:
|
|
tile_size = int(value)
|
|
except ValueError as exc:
|
|
raise SystemExit(f"Invalid tile size: {value}") from exc
|
|
if tile_size < 64:
|
|
raise SystemExit(f"Tile size must be at least 64: {value}")
|
|
|
|
for value in tile_overlap_values:
|
|
try:
|
|
tile_overlap = int(value)
|
|
except ValueError as exc:
|
|
raise SystemExit(f"Invalid tile overlap: {value}") from exc
|
|
if tile_overlap < 0:
|
|
raise SystemExit(f"Tile overlap cannot be negative: {value}")
|
|
|
|
for value in threshold_values:
|
|
try:
|
|
threshold = float(value)
|
|
except ValueError as exc:
|
|
raise SystemExit(f"Invalid confidence threshold: {value}") from exc
|
|
if threshold < 0.0 or threshold > 1.0:
|
|
raise SystemExit(f"Confidence threshold must be between 0 and 1: {value}")
|
|
|
|
with open(output_path, "w", encoding="utf-8") as handle:
|
|
for model in model_values:
|
|
for tile_size in tile_size_values:
|
|
for overlap in tile_overlap_values:
|
|
for threshold in threshold_values:
|
|
label = "_".join(
|
|
[
|
|
f"model_{safe_label(model)}",
|
|
f"tile_{safe_label(tile_size)}",
|
|
f"overlap_{safe_label(overlap)}",
|
|
f"threshold_{safe_label(threshold)}",
|
|
]
|
|
)
|
|
handle.write(f"{model}\t{tile_size}\t{overlap}\t{threshold}\t{label}\n")
|
|
PY
|
|
|
|
echo "== GeoIntel detection quality matrix =="
|
|
echo "Base URL: ${BASE_URL}"
|
|
echo "Raster: ${REAL_RASTER_PATH}"
|
|
echo "Reference vector: ${REAL_REFERENCE_VECTOR_PATH}"
|
|
echo "Models: ${model_requests_normalized}"
|
|
echo "Tile sizes: ${tile_sizes_normalized}"
|
|
echo "Tile overlaps: ${tile_overlaps_normalized}"
|
|
echo "Thresholds: ${thresholds_normalized}"
|
|
echo "Output: ${QUALITY_OUTPUT_DIR}"
|
|
|
|
run_index=0
|
|
while IFS=$'\t' read -r model_request tile_size tile_overlap threshold run_label; do
|
|
run_index=$((run_index + 1))
|
|
run_log="${QUALITY_OUTPUT_DIR}/${run_label}.log"
|
|
quality_response="${QUALITY_OUTPUT_DIR}/${run_label}_quality_checks.json"
|
|
detection_run_response="${QUALITY_OUTPUT_DIR}/${run_label}_detection_run.json"
|
|
run_summary="${QUALITY_OUTPUT_DIR}/${run_label}_summary.json"
|
|
model_env="${model_request}"
|
|
if [ "${model_request}" = "__active__" ]; then
|
|
model_env=""
|
|
fi
|
|
|
|
echo "-- Matrix run ${run_index}: model=${model_request} tile=${tile_size} overlap=${tile_overlap} threshold=${threshold} --"
|
|
if ! REAL_PROJECT_NAME="GeoIntel Detection Quality Matrix ${model_request} tile ${tile_size} overlap ${tile_overlap} threshold ${threshold}" \
|
|
REAL_MODEL_ASSET_ID="${model_env}" \
|
|
REAL_TILE_SIZE="${tile_size}" \
|
|
REAL_TILE_OVERLAP="${tile_overlap}" \
|
|
REAL_CONFIDENCE_THRESHOLD="${threshold}" \
|
|
bash scripts/verify_real_data_detection_qa_workflow.sh "${BASE_URL}" "${REAL_RASTER_PATH}" "${REAL_REFERENCE_VECTOR_PATH}" \
|
|
>"${run_log}" 2>&1; then
|
|
echo "Detection quality matrix run failed. Log: ${run_log}" >&2
|
|
tail -n 80 "${run_log}" >&2 || true
|
|
exit 1
|
|
fi
|
|
|
|
project_id="$(sed -n 's/^Project: //p' "${run_log}" | tail -n 1)"
|
|
raster_dataset_id="$(sed -n 's/^Raster dataset: //p' "${run_log}" | tail -n 1)"
|
|
reference_dataset_id="$(sed -n 's/^Reference dataset: //p' "${run_log}" | tail -n 1)"
|
|
selected_model_asset_id="$(sed -n 's/^Model asset: //p' "${run_log}" | tail -n 1)"
|
|
manifest_path="$(sed -n 's/^Manifest: //p' "${run_log}" | tail -n 1)"
|
|
analysis_run_id="$(sed -n 's/^Analysis run: //p' "${run_log}" | tail -n 1)"
|
|
quality_check_id="$(sed -n 's/^Quality check: //p' "${run_log}" | tail -n 1)"
|
|
detection_count="$(sed -n 's/^Detections: //p' "${run_log}" | tail -n 1)"
|
|
export_id="$(sed -n 's/^Detection export: //p' "${run_log}" | tail -n 1)"
|
|
|
|
if [ -z "${project_id}" ] || [ -z "${analysis_run_id}" ] || [ -z "${quality_check_id}" ]; then
|
|
echo "Matrix run did not print project/run/quality ids. Log: ${run_log}" >&2
|
|
exit 1
|
|
fi
|
|
|
|
curl -fsS "${BASE_URL%/}/api/v1/projects/${project_id}/quality-checks?limit=200" > "${quality_response}"
|
|
curl -fsS "${BASE_URL%/}/api/v1/detection/runs/${analysis_run_id}" > "${detection_run_response}"
|
|
|
|
"${PYTHON_BIN}" - \
|
|
"${quality_response}" \
|
|
"${detection_run_response}" \
|
|
"${run_summary}" \
|
|
"${model_request}" \
|
|
"${selected_model_asset_id}" \
|
|
"${tile_size}" \
|
|
"${tile_overlap}" \
|
|
"${threshold}" \
|
|
"${project_id}" \
|
|
"${raster_dataset_id}" \
|
|
"${reference_dataset_id}" \
|
|
"${manifest_path}" \
|
|
"${analysis_run_id}" \
|
|
"${quality_check_id}" \
|
|
"${detection_count}" \
|
|
"${export_id}" \
|
|
"${run_log}" <<'PY'
|
|
import json
|
|
import sys
|
|
|
|
(
|
|
quality_path,
|
|
detection_run_path,
|
|
output_path,
|
|
model_request,
|
|
selected_model_asset_id,
|
|
tile_size,
|
|
tile_overlap,
|
|
threshold,
|
|
project_id,
|
|
raster_dataset_id,
|
|
reference_dataset_id,
|
|
manifest_path,
|
|
analysis_run_id,
|
|
quality_check_id,
|
|
detection_count,
|
|
export_id,
|
|
run_log,
|
|
) = sys.argv[1:18]
|
|
|
|
with open(quality_path, "r", encoding="utf-8") as handle:
|
|
payload = json.load(handle)
|
|
if "data" not in payload:
|
|
raise SystemExit("Quality-check list is not a canonical GeoIntel data envelope")
|
|
with open(detection_run_path, "r", encoding="utf-8") as handle:
|
|
detection_run_payload = json.load(handle)
|
|
if "data" not in detection_run_payload:
|
|
raise SystemExit("Detection run detail is not a canonical GeoIntel data envelope")
|
|
detection_run = detection_run_payload["data"]
|
|
run_result = detection_run.get("result_json") or {}
|
|
items = payload["data"].get("items") or []
|
|
quality_check = next((item for item in items if str(item.get("id")) == quality_check_id), None)
|
|
if quality_check is None:
|
|
raise SystemExit(f"Quality check not found in project list: {quality_check_id}")
|
|
metrics = {
|
|
metric.get("metric_key"): metric.get("metric_value")
|
|
for metric in quality_check.get("metrics", [])
|
|
if metric.get("metric_key")
|
|
}
|
|
findings = quality_check.get("findings_json") or {}
|
|
summary = {
|
|
"model_request": model_request,
|
|
"model_asset_id": selected_model_asset_id,
|
|
"tile_size": int(tile_size),
|
|
"tile_overlap": int(tile_overlap),
|
|
"threshold": float(threshold),
|
|
"project_id": project_id,
|
|
"raster_dataset_id": raster_dataset_id,
|
|
"reference_dataset_id": reference_dataset_id,
|
|
"manifest_path": manifest_path,
|
|
"analysis_run_id": analysis_run_id,
|
|
"quality_check_id": quality_check_id,
|
|
"detection_count": int(detection_count),
|
|
"raw_detection_count": run_result.get("raw_detection_count", int(detection_count)),
|
|
"suppressed_detection_count": run_result.get("suppressed_detection_count", 0),
|
|
"duplicate_iou_threshold": run_result.get("duplicate_iou_threshold"),
|
|
"quality_status": quality_check.get("status"),
|
|
"quality_score": quality_check.get("score"),
|
|
"precision": metrics.get("precision"),
|
|
"recall": metrics.get("recall"),
|
|
"f1_score": metrics.get("f1_score", metrics.get("f1")),
|
|
"mean_iou": metrics.get("mean_iou"),
|
|
"matches": findings.get("matches"),
|
|
"false_positives": findings.get("false_positives"),
|
|
"false_negatives": findings.get("false_negatives"),
|
|
"export_id": export_id,
|
|
"run_log": run_log,
|
|
}
|
|
with open(output_path, "w", encoding="utf-8") as handle:
|
|
json.dump(summary, handle, indent=2, sort_keys=True)
|
|
print(
|
|
"model={model} tile={tile} overlap={overlap} threshold={threshold} detections={detections} raw={raw} suppressed={suppressed} "
|
|
"score={score} precision={precision} recall={recall} f1={f1} matches={matches} fp={fp} fn={fn}".format(
|
|
model=summary["model_asset_id"],
|
|
tile=summary["tile_size"],
|
|
overlap=summary["tile_overlap"],
|
|
threshold=summary["threshold"],
|
|
detections=summary["detection_count"],
|
|
raw=summary["raw_detection_count"],
|
|
suppressed=summary["suppressed_detection_count"],
|
|
score=summary["quality_score"],
|
|
precision=summary["precision"],
|
|
recall=summary["recall"],
|
|
f1=summary["f1_score"],
|
|
matches=summary["matches"],
|
|
fp=summary["false_positives"],
|
|
fn=summary["false_negatives"],
|
|
)
|
|
)
|
|
PY
|
|
done < "${matrix_manifest}"
|
|
|
|
"${PYTHON_BIN}" - "${QUALITY_OUTPUT_DIR}" "${BASE_URL}" "${matrix_manifest}" <<'PY'
|
|
import glob
|
|
import json
|
|
import os
|
|
import sys
|
|
from datetime import datetime, timezone
|
|
|
|
output_dir, base_url, manifest_path = sys.argv[1:4]
|
|
items = []
|
|
for path in sorted(glob.glob(os.path.join(output_dir, "*_summary.json"))):
|
|
if path.endswith("quality_matrix_summary.json"):
|
|
continue
|
|
with open(path, "r", encoding="utf-8") as handle:
|
|
items.append(json.load(handle))
|
|
|
|
if not items:
|
|
raise SystemExit("No detection quality matrix summaries were produced")
|
|
|
|
|
|
def ranked_with(metric: str):
|
|
return [item for item in items if item.get(metric) is not None]
|
|
|
|
|
|
best_by_score = max(ranked_with("quality_score"), key=lambda item: item["quality_score"], default=None)
|
|
best_by_recall = max(ranked_with("recall"), key=lambda item: item["recall"], default=None)
|
|
best_by_precision = max(ranked_with("precision"), key=lambda item: item["precision"], default=None)
|
|
summary = {
|
|
"generated_at": datetime.now(timezone.utc).isoformat(),
|
|
"base_url": base_url,
|
|
"request_manifest_path": manifest_path,
|
|
"run_count": len(items),
|
|
"best_by_score": best_by_score,
|
|
"best_by_recall": best_by_recall,
|
|
"best_by_precision": best_by_precision,
|
|
"items": items,
|
|
}
|
|
summary_path = os.path.join(output_dir, "quality_matrix_summary.json")
|
|
with open(summary_path, "w", encoding="utf-8") as handle:
|
|
json.dump(summary, handle, indent=2, sort_keys=True)
|
|
|
|
print("")
|
|
print("Detection quality matrix summary")
|
|
print("model\ttile\toverlap\tthreshold\tdetections\traw\tsuppressed\tscore\tprecision\trecall\tf1\tmatches\tfp\tfn")
|
|
for item in items:
|
|
print(
|
|
"{model_asset_id}\t{tile_size}\t{tile_overlap}\t{threshold:.2f}\t{detection_count}\t{raw_detection_count}\t{suppressed_detection_count}\t{quality_score}\t{precision}\t{recall}\t{f1_score}\t{matches}\t{false_positives}\t{false_negatives}".format(
|
|
**item
|
|
)
|
|
)
|
|
print("")
|
|
print(f"Summary: {summary_path}")
|
|
if best_by_score:
|
|
print(
|
|
"best_by_score model={model_asset_id} tile={tile_size} overlap={tile_overlap} threshold={threshold:.2f} score={quality_score} f1={f1_score}".format(
|
|
**best_by_score
|
|
)
|
|
)
|
|
if best_by_recall:
|
|
print(
|
|
"best_by_recall model={model_asset_id} tile={tile_size} overlap={tile_overlap} threshold={threshold:.2f} recall={recall} detections={detection_count}".format(
|
|
**best_by_recall
|
|
)
|
|
)
|
|
if best_by_precision:
|
|
print(
|
|
"best_by_precision model={model_asset_id} tile={tile_size} overlap={tile_overlap} threshold={threshold:.2f} precision={precision} detections={detection_count}".format(
|
|
**best_by_precision
|
|
)
|
|
)
|
|
PY
|