260 lines
10 KiB
Bash
260 lines
10 KiB
Bash
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
|
|
usage() {
|
|
cat >&2 <<'EOF'
|
|
Usage:
|
|
OPERATOR_SAMPLE_MANIFEST_PATH=storage/operator-data/operator_samples_manifest.json \
|
|
QUALITY_MODEL_ASSET_IDS="yolov8n-building-segmentation-pt yolov8n-pt" \
|
|
QUALITY_TILE_SIZES="512 640" \
|
|
QUALITY_TILE_OVERLAPS="64" \
|
|
QUALITY_THRESHOLDS="0.50 0.15" \
|
|
bash scripts/run_multi_sample_detection_quality_matrix.sh [base_url]
|
|
|
|
Optional environment:
|
|
OPERATOR_SAMPLE_MANIFEST_PATH Manifest from prepare_operator_real_data_samples.py.
|
|
OPERATOR_SAMPLE_SLUGS Optional comma/space separated sample slug filter.
|
|
MULTI_SAMPLE_OUTPUT_DIR Output directory, default: artifacts/detection-quality-matrix/multi-sample/<timestamp>.
|
|
QUALITY_MODEL_ASSET_IDS Forwarded to run_detection_quality_matrix.sh.
|
|
QUALITY_TILE_SIZES Forwarded to run_detection_quality_matrix.sh.
|
|
QUALITY_TILE_OVERLAPS Forwarded to run_detection_quality_matrix.sh.
|
|
QUALITY_THRESHOLDS Forwarded to run_detection_quality_matrix.sh.
|
|
REAL_IOU_THRESHOLD Forwarded to run_detection_quality_matrix.sh.
|
|
|
|
This script does not run inference itself. It repeats the existing real-data
|
|
quality matrix once per documented operator sample and combines the summaries.
|
|
EOF
|
|
}
|
|
|
|
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
|
cd "$ROOT"
|
|
|
|
BASE_URL="${1:-${GE_INTEL_BASE_URL:-http://localhost:1202}}"
|
|
OPERATOR_SAMPLE_MANIFEST_PATH="${OPERATOR_SAMPLE_MANIFEST_PATH:-storage/operator-data/operator_samples_manifest.json}"
|
|
OPERATOR_SAMPLE_SLUGS="${OPERATOR_SAMPLE_SLUGS:-}"
|
|
MULTI_SAMPLE_OUTPUT_DIR="${MULTI_SAMPLE_OUTPUT_DIR:-artifacts/detection-quality-matrix/multi-sample/$(date -u +%Y%m%dT%H%M%SZ)}"
|
|
|
|
if [ "${BASE_URL}" = "-h" ] || [ "${BASE_URL}" = "--help" ]; then
|
|
usage
|
|
exit 0
|
|
fi
|
|
|
|
if [ ! -f "${OPERATOR_SAMPLE_MANIFEST_PATH}" ]; then
|
|
echo "OPERATOR_SAMPLE_MANIFEST_PATH does not point to a readable manifest: ${OPERATOR_SAMPLE_MANIFEST_PATH}" >&2
|
|
exit 2
|
|
fi
|
|
|
|
if [ -n "${PYTHON_BIN:-}" ]; then
|
|
PYTHON_BIN="${PYTHON_BIN}"
|
|
else
|
|
PYTHON_BIN=""
|
|
for candidate in python3 python.exe python; do
|
|
if command -v "${candidate}" >/dev/null 2>&1 && "${candidate}" -c "import json, sys" >/dev/null 2>&1; then
|
|
PYTHON_BIN="${candidate}"
|
|
break
|
|
fi
|
|
done
|
|
fi
|
|
|
|
if [ -z "${PYTHON_BIN}" ]; then
|
|
echo "A Python interpreter is required for JSON parsing" >&2
|
|
exit 1
|
|
fi
|
|
|
|
mkdir -p "${MULTI_SAMPLE_OUTPUT_DIR}"
|
|
sample_manifest_tsv="${MULTI_SAMPLE_OUTPUT_DIR}/multi_sample_requests.tsv"
|
|
|
|
"${PYTHON_BIN}" - \
|
|
"${ROOT}" \
|
|
"${OPERATOR_SAMPLE_MANIFEST_PATH}" \
|
|
"${OPERATOR_SAMPLE_SLUGS}" \
|
|
"${sample_manifest_tsv}" <<'PY'
|
|
import json
|
|
import os
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
root = Path(sys.argv[1]).resolve()
|
|
manifest_path = Path(sys.argv[2])
|
|
slug_filter_raw = sys.argv[3]
|
|
output_path = Path(sys.argv[4])
|
|
|
|
payload = json.loads(manifest_path.read_text(encoding="utf-8-sig"))
|
|
samples = payload.get("samples") or []
|
|
if not samples:
|
|
raise SystemExit("Operator sample manifest contains no samples")
|
|
|
|
requested_slugs = {
|
|
value.strip().lower()
|
|
for value in slug_filter_raw.replace(",", " ").split()
|
|
if value.strip()
|
|
}
|
|
|
|
|
|
def resolve_path(raw: str) -> str:
|
|
path = Path(raw)
|
|
if path.exists():
|
|
return str(path)
|
|
if raw.startswith("/app/"):
|
|
candidate = root / raw.removeprefix("/app/")
|
|
if candidate.exists():
|
|
return str(candidate)
|
|
candidate = root / raw
|
|
if candidate.exists():
|
|
return str(candidate)
|
|
raise SystemExit(f"Sample file is not readable from this host: {raw}")
|
|
|
|
|
|
with output_path.open("w", encoding="utf-8") as handle:
|
|
selected = 0
|
|
for sample in samples:
|
|
sample_slug = str(sample.get("sample_slug") or "").lower()
|
|
if not sample_slug:
|
|
raise SystemExit("Operator sample is missing sample_slug")
|
|
if requested_slugs and sample_slug not in requested_slugs:
|
|
continue
|
|
raster_path = resolve_path(str(sample.get("raster_path") or ""))
|
|
reference_path = resolve_path(str(sample.get("reference_path") or ""))
|
|
reference_count = int(sample.get("reference_feature_count") or 0)
|
|
if reference_count < 1:
|
|
raise SystemExit(f"Operator sample has no reference features: {sample_slug}")
|
|
bbox = sample.get("wgs84_bbox") or []
|
|
if len(bbox) != 4:
|
|
raise SystemExit(f"Operator sample has no valid wgs84_bbox: {sample_slug}")
|
|
bbox_csv = ",".join(str(float(value)) for value in bbox)
|
|
municipality = str(sample.get("municipality") or "")
|
|
handle.write(
|
|
f"{sample_slug}\t{raster_path}\t{reference_path}\t{reference_count}\t"
|
|
f"{bbox_csv}\t{municipality}\n"
|
|
)
|
|
selected += 1
|
|
|
|
if selected == 0:
|
|
raise SystemExit("No operator samples matched OPERATOR_SAMPLE_SLUGS")
|
|
PY
|
|
|
|
echo "== GeoIntel multi-sample detection quality matrix =="
|
|
echo "Base URL: ${BASE_URL}"
|
|
echo "Manifest: ${OPERATOR_SAMPLE_MANIFEST_PATH}"
|
|
echo "Sample filter: ${OPERATOR_SAMPLE_SLUGS:-all}"
|
|
echo "Output: ${MULTI_SAMPLE_OUTPUT_DIR}"
|
|
|
|
while IFS=$'\t' read -r sample_slug raster_path reference_path reference_feature_count wgs84_bbox municipality; do
|
|
sample_output_dir="${MULTI_SAMPLE_OUTPUT_DIR}/${sample_slug}"
|
|
mkdir -p "${sample_output_dir}"
|
|
echo "-- Sample ${sample_slug}: reference_features=${reference_feature_count} --"
|
|
project_region="Kempen"
|
|
if [ "${municipality,,}" = "mol" ]; then
|
|
project_region="Mol, Kempen"
|
|
fi
|
|
REAL_RASTER_PATH="${raster_path}" \
|
|
REAL_REFERENCE_VECTOR_PATH="${reference_path}" \
|
|
QUALITY_SAMPLE_SLUG="${sample_slug}" \
|
|
REAL_PROJECT_REGION="${project_region}" \
|
|
REAL_AREA_NAME="${sample_slug} AOI" \
|
|
REAL_AREA_BBOX="${wgs84_bbox}" \
|
|
QUALITY_OUTPUT_DIR="${sample_output_dir}" \
|
|
bash scripts/run_detection_quality_matrix.sh "${BASE_URL}"
|
|
done < "${sample_manifest_tsv}"
|
|
|
|
"${PYTHON_BIN}" - "${MULTI_SAMPLE_OUTPUT_DIR}" "${BASE_URL}" "${OPERATOR_SAMPLE_MANIFEST_PATH}" <<'PY'
|
|
import glob
|
|
import json
|
|
import os
|
|
import sys
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
|
|
output_dir = Path(sys.argv[1])
|
|
base_url = sys.argv[2]
|
|
manifest_path = Path(sys.argv[3])
|
|
manifest_payload = json.loads(manifest_path.read_text(encoding="utf-8-sig"))
|
|
manifest_samples = {
|
|
str(sample.get("sample_slug") or "").lower(): sample
|
|
for sample in manifest_payload.get("samples") or []
|
|
}
|
|
|
|
sample_summaries = []
|
|
flat_items = []
|
|
for summary_path in sorted(glob.glob(str(output_dir / "*" / "quality_matrix_summary.json"))):
|
|
sample_slug = Path(summary_path).parent.name
|
|
sample_metadata = manifest_samples.get(sample_slug, {})
|
|
summary = json.loads(Path(summary_path).read_text(encoding="utf-8"))
|
|
items = summary.get("items") or []
|
|
for item in items:
|
|
enriched = dict(item)
|
|
enriched["sample_slug"] = sample_slug
|
|
enriched["sample_display_name"] = sample_metadata.get("display_name")
|
|
enriched["municipality"] = sample_metadata.get("municipality")
|
|
enriched["operational_zone"] = sample_metadata.get("operational_zone")
|
|
enriched["recommended_split"] = sample_metadata.get("recommended_split")
|
|
enriched["manifest_reference_feature_count"] = sample_metadata.get("reference_feature_count")
|
|
enriched["wgs84_bbox"] = sample_metadata.get("wgs84_bbox")
|
|
flat_items.append(enriched)
|
|
sample_summaries.append(
|
|
{
|
|
"sample_slug": sample_slug,
|
|
"display_name": sample_metadata.get("display_name"),
|
|
"municipality": sample_metadata.get("municipality"),
|
|
"operational_zone": sample_metadata.get("operational_zone"),
|
|
"wgs84_bbox": sample_metadata.get("wgs84_bbox"),
|
|
"recommended_split": sample_metadata.get("recommended_split"),
|
|
"summary_path": summary_path,
|
|
"run_count": len(items),
|
|
"best_by_score": summary.get("best_by_score"),
|
|
"best_by_recall": summary.get("best_by_recall"),
|
|
"best_by_precision": summary.get("best_by_precision"),
|
|
}
|
|
)
|
|
|
|
if not flat_items:
|
|
raise SystemExit("No sample quality matrix summaries were produced")
|
|
|
|
|
|
def best(metric: str):
|
|
ranked = [item for item in flat_items if item.get(metric) is not None]
|
|
return max(ranked, key=lambda item: item[metric], default=None)
|
|
|
|
|
|
best_by_sample = {
|
|
sample["sample_slug"]: sample.get("best_by_score")
|
|
for sample in sample_summaries
|
|
}
|
|
summary = {
|
|
"generated_at": datetime.now(timezone.utc).isoformat(),
|
|
"base_url": base_url,
|
|
"operator_sample_manifest_path": str(manifest_path),
|
|
"sample_count": len(sample_summaries),
|
|
"run_count": len(flat_items),
|
|
"best_overall_by_score": best("quality_score"),
|
|
"best_overall_by_recall": best("recall"),
|
|
"best_overall_by_precision": best("precision"),
|
|
"best_by_sample": best_by_sample,
|
|
"sample_summaries": sample_summaries,
|
|
"items": flat_items,
|
|
}
|
|
summary_path = output_dir / "multi_sample_quality_summary.json"
|
|
summary_path.write_text(json.dumps(summary, indent=2, sort_keys=True), encoding="utf-8")
|
|
|
|
print("")
|
|
print("Multi-sample detection quality summary")
|
|
print("sample\tzone\tmodel\ttile\toverlap\tthreshold\tdetections\traw\tsuppressed\tscore\tprecision\trecall\tf1\tmatches\tfp\tfn\tcoverage\tdiagnostic_gap")
|
|
for item in flat_items:
|
|
print(
|
|
"{sample_slug}\t{operational_zone}\t{model_asset_id}\t{tile_size}\t{tile_overlap}\t{threshold:.2f}\t{detection_count}\t{raw_detection_count}\t{suppressed_detection_count}\t{quality_score}\t{precision}\t{recall}\t{f1_score}\t{matches}\t{false_positives}\t{false_negatives}\t{reference_coverage_ratio}\t{possible_box_to_footprint_mismatch_count}".format(
|
|
**item
|
|
)
|
|
)
|
|
print("")
|
|
print(f"Summary: {summary_path}")
|
|
for key in ("best_overall_by_score", "best_overall_by_recall", "best_overall_by_precision"):
|
|
item = summary.get(key)
|
|
if item:
|
|
print(
|
|
f"{key} sample={item['sample_slug']} model={item['model_asset_id']} "
|
|
f"tile={item['tile_size']} overlap={item['tile_overlap']} "
|
|
f"threshold={item['threshold']:.2f} score={item.get('quality_score')} "
|
|
f"precision={item.get('precision')} recall={item.get('recall')} f1={item.get('f1_score')}"
|
|
)
|
|
PY
|