diff --git a/CHANGELOG.md b/CHANGELOG.md index 47def051..cf9faa06 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,13 @@ # Changelog +## Sprint 124 Detection calibration sweep tooling (2026-07-07) + +- Added `scripts/run_detection_calibration_sweep.sh` to run the existing real-data detection + QA workflow across multiple configured-YOLO confidence thresholds. +- The sweep writes per-threshold logs and a `calibration_summary.json` with persisted detection count, QA score, precision, recall, F1, mean IoU, matches, false positives and false negatives. +- Added readiness syntax coverage and static regression coverage for the calibration sweep contract. +- No new model dependencies, provider fetching, fake detections, API contracts or product UI behavior were introduced. + ## Sprint 123 YOLO class and tile CRS normalization (2026-07-07) - Fixed configured-YOLO class filtering so model labels such as `Building` match operator/domain filters such as `building`. diff --git a/backend/README.md b/backend/README.md index bf894b8c..6583560b 100644 --- a/backend/README.md +++ b/backend/README.md @@ -374,6 +374,21 @@ manifests generated for AI handoff include source CRS metadata so pixel-space model outputs can be transformed to WGS84 GeoJSON coordinates. Current V1 upload support is limited to GeoTIFF-style rasters and GeoJSON/JSON reference vectors. +For model-quality calibration, run the confidence sweep wrapper: + +```bash +REAL_RASTER_PATH=/mnt/user/appdata/geointel/storage/operator-data/geel_orthophoto_wms_512.tif \ +REAL_REFERENCE_VECTOR_PATH=/mnt/user/appdata/geointel/storage/operator-data/geel_grb_gbg_buildings.geojson \ +CALIBRATION_THRESHOLDS="0.50 0.35 0.25 0.15" \ +bash scripts/run_detection_calibration_sweep.sh http://192.168.10.150:1202 +``` + +The sweep creates one real persisted workflow run per threshold, fetches the +persisted `QualityCheck`/`Metric` rows and writes a `calibration_summary.json` +with detection count, score, precision, recall, F1, mean IoU and false +positive/negative counts. It is intended to tune confidence/IoU/model choices, +not to add new inference behavior. + ### Run backend ```bash diff --git a/backend/tests/test_sprint124_detection_calibration_sweep.py b/backend/tests/test_sprint124_detection_calibration_sweep.py new file mode 100644 index 00000000..24588218 --- /dev/null +++ b/backend/tests/test_sprint124_detection_calibration_sweep.py @@ -0,0 +1,29 @@ +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[2] + + +def test_detection_calibration_sweep_reuses_real_data_workflow_and_reports_qa_metrics() -> None: + script_path = ROOT / "scripts" / "run_detection_calibration_sweep.sh" + readiness = (ROOT / "scripts" / "run_readiness_check.sh").read_text(encoding="utf-8") + + assert script_path.exists() + script = script_path.read_text(encoding="utf-8") + + assert "bash -n scripts/run_detection_calibration_sweep.sh" in readiness + assert "verify_real_data_detection_qa_workflow.sh" in script + assert "CALIBRATION_THRESHOLDS" in script + assert "REAL_CONFIDENCE_THRESHOLD" in script + assert "REAL_RASTER_PATH" in script + assert "REAL_REFERENCE_VECTOR_PATH" in script + assert "/api/v1/projects/${project_id}/quality-checks" in script + assert "quality_check_id" in script + assert "false_positives" in script + assert "false_negatives" in script + assert "quality_score" in script + assert "best_by_score" in script + assert "calibration_summary.json" in script + assert "demo/workflow" not in script + assert "fixture_mode" not in script + assert "will_download_models" not in script diff --git a/docs/AI_PIPELINES.md b/docs/AI_PIPELINES.md index 6468e46b..07819d35 100644 --- a/docs/AI_PIPELINES.md +++ b/docs/AI_PIPELINES.md @@ -167,6 +167,21 @@ download model weights. A zero detection count is valid as runtime evidence only when the selected model genuinely returns no usable detections after canonical class filtering; it does not prove the model is useful for the target imagery. +For confidence-threshold calibration, use the sweep wrapper: + +```bash +REAL_RASTER_PATH=/mnt/user/appdata/geointel/storage/operator-data/geel_orthophoto_wms_512.tif \ +REAL_REFERENCE_VECTOR_PATH=/mnt/user/appdata/geointel/storage/operator-data/geel_grb_gbg_buildings.geojson \ +CALIBRATION_THRESHOLDS="0.50 0.35 0.25 0.15" \ +bash scripts/run_detection_calibration_sweep.sh http://192.168.10.150:1202 +``` + +The sweep runs the real-data workflow once per threshold and then reads the +persisted project quality-check list to build `calibration_summary.json`. +Results are honest QA/QC evidence from persisted detections and persisted +reference `vector_features`; no demo detections, live provider fetches or model +downloads are introduced by the calibration tool. + ### Sprint 8C detection visualization and QA status Sprint 8C makes persisted detections reviewable: diff --git a/docs/CODEX_EXECUTION_LOG.md b/docs/CODEX_EXECUTION_LOG.md index 222f5bae..d9e9ebcd 100644 --- a/docs/CODEX_EXECUTION_LOG.md +++ b/docs/CODEX_EXECUTION_LOG.md @@ -1,3 +1,30 @@ +## Sprint 124 Detection calibration sweep tooling (2026-07-07) + +Changed: +- Added `scripts/run_detection_calibration_sweep.sh` as an operator-facing confidence-threshold sweep for the configured-YOLO real-data path. +- The sweep reuses `scripts/verify_real_data_detection_qa_workflow.sh` once per threshold, so each row is backed by persisted Project, Dataset, AnalysisRun, Detection, QualityCheck, Metric and export records. +- The sweep fetches project `quality-checks` after each run and writes per-threshold summaries plus `calibration_summary.json` with detection count, QA score, precision, recall, F1, mean IoU, matches, false positives and false negatives. +- Registered the sweep in `scripts/run_readiness_check.sh` as a syntax check. +- Added regression coverage in `backend/tests/test_sprint124_detection_calibration_sweep.py`. +- Updated `scripts/README.md`, `backend/README.md`, `docs/AI_PIPELINES.md`, `docs/TODO.md` and `CHANGELOG.md`. + +Validation: +- RED: `python -m pytest backend/tests/test_sprint124_detection_calibration_sweep.py -q` failed because `scripts/run_detection_calibration_sweep.sh` did not exist. +- `python -m pytest backend/tests/test_sprint124_detection_calibration_sweep.py -q` passed. +- `bash -n scripts/run_detection_calibration_sweep.sh` passed. +- `bash scripts/run_detection_calibration_sweep.sh --help` passed. +- Missing-input guard printed usage and did not start a live workflow. + +Open: +- Run the sweep on Tower with the Geel operator sample and record the observed threshold metrics. + +Limitations: +- The sweep is intentionally mutating and creates one real workflow run per threshold. +- It is calibration tooling only; it does not change inference, add model downloads, fetch providers, seed demo detections or change API/UI behavior. + +Next recommended pass: +- Run the Tower sweep for the current Geel sample, then repeat on additional operator-provided orthophoto/reference samples before choosing V1 default confidence/IoU guidance. + ## Sprint 123 YOLO class and tile CRS normalization (2026-07-07) Changed: diff --git a/docs/TODO.md b/docs/TODO.md index 6eba12a5..9c558307 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -95,6 +95,7 @@ This file now starts with the current implementation status. Older preparation/b - [x] Validate the configured building model on a real georeferenced Kempen orthophoto/GeoTIFF with persisted reference vectors and QA/QC metrics. - [x] Fix configured-YOLO mixed-case class labels so `Building` model output matches `building` domain filters. - [x] Persist CRS metadata in raster tile manifests so AI detections can be transformed to WGS84 GeoJSON correctly. +- [x] Add real-data detection calibration sweep tooling for confidence-threshold and QA/QC metric comparison. - [ ] Calibrate confidence, IoU and model selection against persisted Geel detections and additional local orthophoto/reference samples. ## Sprint 8 status diff --git a/scripts/README.md b/scripts/README.md index b5adce3f..f3cd232a 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -189,6 +189,24 @@ operationally only when the selected model genuinely returns no usable detections after class filtering; it must be interpreted as model/data quality evidence rather than as a successful building extraction result. +Run a confidence-threshold calibration sweep against the same real-data path: + +```bash +REAL_RASTER_PATH=/mnt/user/appdata/geointel/storage/operator-data/geel_orthophoto_wms_512.tif \ +REAL_REFERENCE_VECTOR_PATH=/mnt/user/appdata/geointel/storage/operator-data/geel_grb_gbg_buildings.geojson \ +CALIBRATION_THRESHOLDS="0.50 0.35 0.25 0.15" \ +bash scripts/run_detection_calibration_sweep.sh http://192.168.10.150:1202 +``` + +The sweep reuses `verify_real_data_detection_qa_workflow.sh` once per +threshold, so every row is backed by persisted Project, Dataset, AnalysisRun, +Detection, QualityCheck, Metric and export records. It writes per-threshold +logs plus `calibration_summary.json` under +`artifacts/detection-calibration/` unless +`CALIBRATION_OUTPUT_DIR` is set. This is a calibration/benchmarking tool only: +it does not seed demo data, enable fixture detections, fetch external data or +download model weights. + Docker images install only the GIS runtime by default. To build a local/Tower image with PyTorch/Ultralytics available for the configured-YOLO preflight and runtime path, set: diff --git a/scripts/run_detection_calibration_sweep.sh b/scripts/run_detection_calibration_sweep.sh new file mode 100644 index 00000000..b0e4b0c7 --- /dev/null +++ b/scripts/run_detection_calibration_sweep.sh @@ -0,0 +1,235 @@ +#!/usr/bin/env bash +set -euo pipefail + +usage() { + cat >&2 <<'EOF' +Usage: + REAL_RASTER_PATH=/path/to/orthophoto.tif \ + REAL_REFERENCE_VECTOR_PATH=/path/to/reference-buildings.geojson \ + CALIBRATION_THRESHOLDS="0.50 0.35 0.25 0.15" \ + bash scripts/run_detection_calibration_sweep.sh [base_url] + +or: + bash scripts/run_detection_calibration_sweep.sh [base_url] /path/to/orthophoto.tif /path/to/reference-buildings.geojson + +Required inputs: + REAL_RASTER_PATH Georeferenced .tif/.tiff/.geotiff raster. + REAL_REFERENCE_VECTOR_PATH EPSG-aware .geojson/.json reference vector with building polygons. + +Optional environment: + CALIBRATION_THRESHOLDS Space/comma separated confidence thresholds, default: 0.50 0.35 0.25 0.15. + CALIBRATION_OUTPUT_DIR Output directory, default: artifacts/detection-calibration/. + REAL_MODEL_ASSET_ID Specific /api/v1/detection/model-assets id to use. + REAL_TILE_SIZE Raster tile size passed to the underlying real-data workflow. + REAL_TILE_OVERLAP Raster tile overlap passed to the underlying real-data workflow. + REAL_IOU_THRESHOLD QA IoU threshold, default inherited by the underlying workflow. +EOF +} + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$ROOT" + +BASE_URL="${1:-${GE_INTEL_BASE_URL:-http://localhost:1202}}" +REAL_RASTER_PATH="${2:-${REAL_RASTER_PATH:-}}" +REAL_REFERENCE_VECTOR_PATH="${3:-${REAL_REFERENCE_VECTOR_PATH:-}}" +CALIBRATION_THRESHOLDS="${CALIBRATION_THRESHOLDS:-0.50 0.35 0.25 0.15}" +CALIBRATION_OUTPUT_DIR="${CALIBRATION_OUTPUT_DIR:-artifacts/detection-calibration/$(date -u +%Y%m%dT%H%M%SZ)}" + +if [ "${BASE_URL}" = "-h" ] || [ "${BASE_URL}" = "--help" ]; then + usage + exit 0 +fi + +if [ -z "${REAL_RASTER_PATH}" ] || [ -z "${REAL_REFERENCE_VECTOR_PATH}" ]; then + usage + exit 2 +fi + +if ! command -v curl >/dev/null 2>&1; then + echo "curl is required for detection calibration sweep verification" >&2 + exit 1 +fi + +if [ -n "${PYTHON_BIN:-}" ]; then + PYTHON_BIN="${PYTHON_BIN}" +else + PYTHON_BIN="" + for candidate in python3 python.exe python; do + if command -v "${candidate}" >/dev/null 2>&1 && "${candidate}" -c "import json, sys" >/dev/null 2>&1; then + PYTHON_BIN="${candidate}" + break + fi + done +fi + +if [ -z "${PYTHON_BIN}" ]; then + echo "A Python interpreter is required for JSON parsing" >&2 + exit 1 +fi + +thresholds_normalized="$(printf '%s' "${CALIBRATION_THRESHOLDS}" | tr ',' ' ')" +mkdir -p "${CALIBRATION_OUTPUT_DIR}" + +"${PYTHON_BIN}" - "${thresholds_normalized}" <<'PY' +import sys + +raw = sys.argv[1].split() +if not raw: + raise SystemExit("CALIBRATION_THRESHOLDS must contain at least one threshold") +for value in raw: + try: + threshold = float(value) + except ValueError as exc: + raise SystemExit(f"Invalid confidence threshold: {value}") from exc + if threshold < 0.0 or threshold > 1.0: + raise SystemExit(f"Confidence threshold must be between 0 and 1: {value}") +PY + +echo "== GeoIntel detection calibration sweep ==" +echo "Base URL: ${BASE_URL}" +echo "Raster: ${REAL_RASTER_PATH}" +echo "Reference vector: ${REAL_REFERENCE_VECTOR_PATH}" +echo "Thresholds: ${thresholds_normalized}" +echo "Output: ${CALIBRATION_OUTPUT_DIR}" + +run_index=0 +for threshold in ${thresholds_normalized}; do + run_index=$((run_index + 1)) + threshold_label="$(printf '%s' "${threshold}" | tr '.-' 'pm')" + run_log="${CALIBRATION_OUTPUT_DIR}/threshold_${threshold_label}.log" + quality_response="${CALIBRATION_OUTPUT_DIR}/threshold_${threshold_label}_quality_checks.json" + run_summary="${CALIBRATION_OUTPUT_DIR}/threshold_${threshold_label}_summary.json" + + echo "-- Threshold ${threshold} (${run_index}) --" + if ! REAL_PROJECT_NAME="GeoIntel Detection Calibration ${threshold}" \ + REAL_CONFIDENCE_THRESHOLD="${threshold}" \ + bash scripts/verify_real_data_detection_qa_workflow.sh "${BASE_URL}" "${REAL_RASTER_PATH}" "${REAL_REFERENCE_VECTOR_PATH}" \ + >"${run_log}" 2>&1; then + echo "Calibration threshold ${threshold} failed. Log: ${run_log}" >&2 + tail -n 80 "${run_log}" >&2 || true + exit 1 + fi + + project_id="$(sed -n 's/^Project: //p' "${run_log}" | tail -n 1)" + analysis_run_id="$(sed -n 's/^Analysis run: //p' "${run_log}" | tail -n 1)" + quality_check_id="$(sed -n 's/^Quality check: //p' "${run_log}" | tail -n 1)" + detection_count="$(sed -n 's/^Detections: //p' "${run_log}" | tail -n 1)" + + if [ -z "${project_id}" ] || [ -z "${analysis_run_id}" ] || [ -z "${quality_check_id}" ]; then + echo "Calibration threshold ${threshold} did not print project/run/quality ids. Log: ${run_log}" >&2 + exit 1 + fi + + curl -fsS "${BASE_URL%/}/api/v1/projects/${project_id}/quality-checks?limit=200" > "${quality_response}" + + "${PYTHON_BIN}" - \ + "${quality_response}" \ + "${run_summary}" \ + "${threshold}" \ + "${project_id}" \ + "${analysis_run_id}" \ + "${quality_check_id}" \ + "${detection_count}" \ + "${run_log}" <<'PY' +import json +import sys + +quality_path, output_path, threshold, project_id, analysis_run_id, quality_check_id, detection_count, run_log = sys.argv[1:9] +with open(quality_path, "r", encoding="utf-8") as handle: + payload = json.load(handle) +if "data" not in payload: + raise SystemExit("Quality-check list is not a canonical GeoIntel data envelope") +items = payload["data"].get("items") or [] +quality_check = next((item for item in items if str(item.get("id")) == quality_check_id), None) +if quality_check is None: + raise SystemExit(f"Quality check not found in project list: {quality_check_id}") +metrics = { + metric.get("metric_key"): metric.get("metric_value") + for metric in quality_check.get("metrics", []) + if metric.get("metric_key") +} +findings = quality_check.get("findings_json") or {} +summary = { + "threshold": float(threshold), + "project_id": project_id, + "analysis_run_id": analysis_run_id, + "quality_check_id": quality_check_id, + "detection_count": int(detection_count), + "quality_status": quality_check.get("status"), + "quality_score": quality_check.get("score"), + "precision": metrics.get("precision"), + "recall": metrics.get("recall"), + "f1_score": metrics.get("f1_score"), + "mean_iou": metrics.get("mean_iou"), + "matches": findings.get("matches"), + "false_positives": findings.get("false_positives"), + "false_negatives": findings.get("false_negatives"), + "run_log": run_log, +} +with open(output_path, "w", encoding="utf-8") as handle: + json.dump(summary, handle, indent=2, sort_keys=True) +print( + "threshold={threshold} detections={detections} score={score} " + "precision={precision} recall={recall} f1={f1} matches={matches} fp={fp} fn={fn}".format( + threshold=summary["threshold"], + detections=summary["detection_count"], + score=summary["quality_score"], + precision=summary["precision"], + recall=summary["recall"], + f1=summary["f1_score"], + matches=summary["matches"], + fp=summary["false_positives"], + fn=summary["false_negatives"], + ) +) +PY +done + +"${PYTHON_BIN}" - "${CALIBRATION_OUTPUT_DIR}" "${BASE_URL}" "${thresholds_normalized}" <<'PY' +import glob +import json +import os +import sys +from datetime import datetime, timezone + +output_dir, base_url, thresholds = sys.argv[1:4] +items = [] +for path in sorted(glob.glob(os.path.join(output_dir, "threshold_*_summary.json"))): + with open(path, "r", encoding="utf-8") as handle: + items.append(json.load(handle)) + +ranked = [ + item + for item in items + if item.get("quality_score") is not None +] +best_by_score = max(ranked, key=lambda item: item["quality_score"], default=None) +summary = { + "generated_at": datetime.now(timezone.utc).isoformat(), + "base_url": base_url, + "thresholds": [float(value) for value in thresholds.split()], + "best_by_score": best_by_score, + "items": items, +} +summary_path = os.path.join(output_dir, "calibration_summary.json") +with open(summary_path, "w", encoding="utf-8") as handle: + json.dump(summary, handle, indent=2, sort_keys=True) + +print("") +print("Detection calibration summary") +print("threshold\tdetections\tscore\tprecision\trecall\tf1\tmatches\tfp\tfn") +for item in items: + print( + "{threshold:.2f}\t{detection_count}\t{quality_score}\t{precision}\t{recall}\t{f1_score}\t{matches}\t{false_positives}\t{false_negatives}".format( + **item + ) + ) +print("") +print(f"Summary: {summary_path}") +if best_by_score: + print( + "best_by_score threshold={threshold:.2f} score={quality_score} f1={f1_score} detections={detection_count}".format( + **best_by_score + ) + ) +PY diff --git a/scripts/run_readiness_check.sh b/scripts/run_readiness_check.sh index 71b2b8a5..a7a7725a 100755 --- a/scripts/run_readiness_check.sh +++ b/scripts/run_readiness_check.sh @@ -55,6 +55,7 @@ bash -n scripts/verify_demo_raster_workflow.sh bash -n scripts/verify_ai_handoff_interactions.sh bash -n scripts/verify_model_asset_detection_workflow.sh bash -n scripts/verify_real_data_detection_qa_workflow.sh +bash -n scripts/run_detection_calibration_sweep.sh bash -n scripts/verify_workbench_default_state.sh bash -n scripts/verify_workbench_interactions.sh bash -n scripts/verify_gis_runtime.sh