diff --git a/CHANGELOG.md b/CHANGELOG.md index 1143dbb3..01d93ec5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,13 @@ # Changelog +## Sprint 126 Detection quality matrix tooling (2026-07-07) + +- Added `scripts/run_detection_quality_matrix.sh` to compare local model assets, raster tile sizes, tile overlaps and confidence thresholds through the existing real-data detection + QA workflow. +- The script writes per-run logs and a `quality_matrix_summary.json` with detection count, QA score, precision, recall, F1, mean IoU, matches, false positives and false negatives. +- The summary ranks `best_by_score`, `best_by_recall` and `best_by_precision` for operator model-quality decisions. +- Added readiness syntax coverage and static regression coverage for the quality matrix contract. + ## Sprint 125 Detection calibration evidence bundle (2026-07-07) - Added `scripts/export_detection_calibration_evidence.sh` to export persisted QA evidence from a detection calibration summary. diff --git a/backend/README.md b/backend/README.md index 97bb9d99..8ba9b968 100644 --- a/backend/README.md +++ b/backend/README.md @@ -389,6 +389,26 @@ with detection count, score, precision, recall, F1, mean IoU and false positive/negative counts. It is intended to tune confidence/IoU/model choices, not to add new inference behavior. +To compare local model assets and tile settings as well as thresholds, run the +quality matrix wrapper: + +```bash +REAL_RASTER_PATH=/mnt/user/appdata/geointel/storage/operator-data/geel_orthophoto_wms_512.tif \ +REAL_REFERENCE_VECTOR_PATH=/mnt/user/appdata/geointel/storage/operator-data/geel_grb_gbg_buildings.geojson \ +QUALITY_MODEL_ASSET_IDS="yolov8n-building-segmentation-pt yolov8n-pt" \ +QUALITY_TILE_SIZES="512 640" \ +QUALITY_TILE_OVERLAPS="64" \ +QUALITY_THRESHOLDS="0.50 0.15" \ +bash scripts/run_detection_quality_matrix.sh http://192.168.10.150:1202 +``` + +The matrix creates one real persisted workflow run per combination and writes +`quality_matrix_summary.json` with the selected model asset, tile size, tile +overlap, threshold, detection count, QA score, precision, recall, F1, mean IoU +and false-positive/false-negative counts. It ranks `best_by_score`, +`best_by_recall` and `best_by_precision`. It does not download weights, create +fake detections, fetch live providers or change backend API behavior. + To inspect the evidence behind a calibration run, export the persisted QA evidence bundle: diff --git a/backend/tests/test_sprint126_detection_quality_matrix.py b/backend/tests/test_sprint126_detection_quality_matrix.py new file mode 100644 index 00000000..f73f657f --- /dev/null +++ b/backend/tests/test_sprint126_detection_quality_matrix.py @@ -0,0 +1,36 @@ +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[2] + + +def test_detection_quality_matrix_compares_models_tiles_and_thresholds() -> None: + script_path = ROOT / "scripts" / "run_detection_quality_matrix.sh" + readiness = (ROOT / "scripts" / "run_readiness_check.sh").read_text(encoding="utf-8") + + assert script_path.exists() + script = script_path.read_text(encoding="utf-8") + + assert "bash -n scripts/run_detection_quality_matrix.sh" in readiness + assert "verify_real_data_detection_qa_workflow.sh" in script + assert "QUALITY_MODEL_ASSET_IDS" in script + assert "QUALITY_TILE_SIZES" in script + assert "QUALITY_TILE_OVERLAPS" in script + assert "QUALITY_THRESHOLDS" in script + assert "REAL_MODEL_ASSET_ID" in script + assert "REAL_TILE_SIZE" in script + assert "REAL_TILE_OVERLAP" in script + assert "REAL_CONFIDENCE_THRESHOLD" in script + assert "REAL_RASTER_PATH" in script + assert "REAL_REFERENCE_VECTOR_PATH" in script + assert "/api/v1/projects/${project_id}/quality-checks" in script + assert "quality_matrix_summary.json" in script + assert "best_by_score" in script + assert "best_by_recall" in script + assert "best_by_precision" in script + assert "quality_score" in script + assert "false_positives" in script + assert "false_negatives" in script + assert "demo/workflow" not in script + assert "fixture_mode" not in script + assert "will_download_models" not in script diff --git a/docs/AI_PIPELINES.md b/docs/AI_PIPELINES.md index cdd303ec..ce26f95d 100644 --- a/docs/AI_PIPELINES.md +++ b/docs/AI_PIPELINES.md @@ -182,6 +182,25 @@ Results are honest QA/QC evidence from persisted detections and persisted reference `vector_features`; no demo detections, live provider fetches or model downloads are introduced by the calibration tool. +For model/tile/threshold selection, use the quality matrix wrapper: + +```bash +REAL_RASTER_PATH=/mnt/user/appdata/geointel/storage/operator-data/geel_orthophoto_wms_512.tif \ +REAL_REFERENCE_VECTOR_PATH=/mnt/user/appdata/geointel/storage/operator-data/geel_grb_gbg_buildings.geojson \ +QUALITY_MODEL_ASSET_IDS="yolov8n-building-segmentation-pt yolov8n-pt" \ +QUALITY_TILE_SIZES="512 640" \ +QUALITY_TILE_OVERLAPS="64" \ +QUALITY_THRESHOLDS="0.50 0.15" \ +bash scripts/run_detection_quality_matrix.sh http://192.168.10.150:1202 +``` + +The matrix repeats the same persisted real-data workflow for every combination +and writes `quality_matrix_summary.json` with detection count, QA score, +precision, recall, F1, mean IoU and false-positive/false-negative counts. The +rankings `best_by_score`, `best_by_recall` and `best_by_precision` are operator +decision aids only; GeoIntel still does not download models, seed fixture +detections or treat AI detections as ground truth without QA/QC. + For visual error inspection, export the persisted QA evidence from a calibration summary: diff --git a/docs/CODEX_EXECUTION_LOG.md b/docs/CODEX_EXECUTION_LOG.md index c45fda92..2d6019e7 100644 --- a/docs/CODEX_EXECUTION_LOG.md +++ b/docs/CODEX_EXECUTION_LOG.md @@ -1,3 +1,34 @@ +## Sprint 126 Detection quality matrix tooling (2026-07-07) + +Changed: +- Added `scripts/run_detection_quality_matrix.sh` as an operator-facing model/tile/threshold matrix for the configured-YOLO real-data path. +- The matrix reuses `scripts/verify_real_data_detection_qa_workflow.sh` for each row so every result is backed by persisted Project, Dataset, AnalysisRun, Detection, QualityCheck, Metric and Export records. +- The script accepts `QUALITY_MODEL_ASSET_IDS`, `QUALITY_TILE_SIZES`, `QUALITY_TILE_OVERLAPS` and `QUALITY_THRESHOLDS`, writes per-run logs and produces `quality_matrix_summary.json`. +- The summary reports model asset, tile size, overlap, confidence threshold, detection count, QA score, precision, recall, F1, mean IoU, matches, false positives and false negatives. +- Added `best_by_score`, `best_by_recall` and `best_by_precision` rankings for operator model-quality decisions. +- Added readiness syntax coverage and regression coverage in `backend/tests/test_sprint126_detection_quality_matrix.py`. +- Updated `scripts/README.md`, `backend/README.md`, `docs/AI_PIPELINES.md`, `docs/TODO.md` and `CHANGELOG.md`. + +Tested: +- RED: `python -m pytest backend\tests\test_sprint126_detection_quality_matrix.py -q` failed because `scripts/run_detection_quality_matrix.sh` did not exist. +- `python -m pytest backend\tests\test_sprint126_detection_quality_matrix.py -q` passed. +- `python -m pytest backend\tests\test_sprint126_detection_quality_matrix.py backend\tests\test_sprint124_detection_calibration_sweep.py backend\tests\test_sprint125_detection_calibration_evidence_bundle.py -q` passed. +- `bash -n scripts/run_detection_quality_matrix.sh` passed. +- `bash scripts/run_detection_quality_matrix.sh --help` passed. +- `python scripts\smoke_docs.py` passed. +- `git diff --check` passed. +- `bash scripts/run_readiness_check.sh` passed: 390 backend tests, frontend typecheck/build, Alembic head `202606120900`, live smoke syntax checks and the new matrix syntax check. + +Open: +- Live Tower matrix run still needed against the current Geel operator sample. + +Limitations: +- This is operator benchmarking tooling only. It does not change inference behavior, add model downloads, seed fixture detections, fetch providers, change API contracts or change migrations. +- A single Geel sample is not enough to declare a production V1 building-extraction baseline; additional orthophoto/reference samples are still needed before picking defaults. + +Next recommended pass: +- Run the matrix on Tower for the current Geel operator sample, then decide whether the active building model should remain the default evaluation model or be replaced. + ## Sprint 125 Detection calibration evidence bundle (2026-07-07) Changed: diff --git a/docs/TODO.md b/docs/TODO.md index 67a58ba9..e4d0ee3f 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -97,6 +97,7 @@ This file now starts with the current implementation status. Older preparation/b - [x] Persist CRS metadata in raster tile manifests so AI detections can be transformed to WGS84 GeoJSON correctly. - [x] Add real-data detection calibration sweep tooling for confidence-threshold and QA/QC metric comparison. - [x] Add calibration QA evidence export tooling for false-positive/false-negative inspection artifacts. +- [x] Add real-data detection quality matrix tooling for model/tile/threshold comparison. - [ ] Calibrate confidence, IoU and model selection against persisted Geel detections and additional local orthophoto/reference samples. ## Sprint 8 status diff --git a/scripts/README.md b/scripts/README.md index 01575cde..d099220c 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -207,6 +207,28 @@ logs plus `calibration_summary.json` under it does not seed demo data, enable fixture detections, fetch external data or download model weights. +Run a broader model/tile/threshold quality matrix when multiple local model +assets or tile settings need to be compared: + +```bash +REAL_RASTER_PATH=/mnt/user/appdata/geointel/storage/operator-data/geel_orthophoto_wms_512.tif \ +REAL_REFERENCE_VECTOR_PATH=/mnt/user/appdata/geointel/storage/operator-data/geel_grb_gbg_buildings.geojson \ +QUALITY_MODEL_ASSET_IDS="yolov8n-building-segmentation-pt yolov8n-pt" \ +QUALITY_TILE_SIZES="512 640" \ +QUALITY_TILE_OVERLAPS="64" \ +QUALITY_THRESHOLDS="0.50 0.15" \ +bash scripts/run_detection_quality_matrix.sh http://192.168.10.150:1202 +``` + +The quality matrix repeats the same real-data upload, tiling, configured-YOLO, +QA/QC and export workflow for every model/tile/threshold row. It writes per-run +logs plus `quality_matrix_summary.json` under +`artifacts/detection-quality-matrix/` unless `QUALITY_OUTPUT_DIR` is +set. The summary ranks `best_by_score`, `best_by_recall` and +`best_by_precision` so the next model decision is based on persisted +`QualityCheck`/`Metric` evidence rather than visual guesses. It does not create +provider data, use fixtures or download model weights. + Export calibration QA evidence for visual review: ```bash diff --git a/scripts/run_detection_quality_matrix.sh b/scripts/run_detection_quality_matrix.sh new file mode 100644 index 00000000..8906a1df --- /dev/null +++ b/scripts/run_detection_quality_matrix.sh @@ -0,0 +1,376 @@ +#!/usr/bin/env bash +set -euo pipefail + +usage() { + cat >&2 <<'EOF' +Usage: + REAL_RASTER_PATH=/path/to/orthophoto.tif \ + REAL_REFERENCE_VECTOR_PATH=/path/to/reference-buildings.geojson \ + QUALITY_MODEL_ASSET_IDS="yolov8n-building-segmentation-pt yolov8n-pt" \ + QUALITY_TILE_SIZES="512 640" \ + QUALITY_TILE_OVERLAPS="64" \ + QUALITY_THRESHOLDS="0.50 0.25 0.15" \ + bash scripts/run_detection_quality_matrix.sh [base_url] + +or: + bash scripts/run_detection_quality_matrix.sh [base_url] /path/to/orthophoto.tif /path/to/reference-buildings.geojson + +Required inputs: + REAL_RASTER_PATH Georeferenced .tif/.tiff/.geotiff raster. + REAL_REFERENCE_VECTOR_PATH EPSG-aware .geojson/.json reference vector with building polygons. + +Optional environment: + QUALITY_MODEL_ASSET_IDS Space/comma separated local model asset IDs. Default: active configured model asset. + QUALITY_TILE_SIZES Space/comma separated raster tile sizes, default: 640. + QUALITY_TILE_OVERLAPS Space/comma separated raster tile overlaps, default: 64. + QUALITY_THRESHOLDS Space/comma separated confidence thresholds, default: 0.50 0.25 0.15. + QUALITY_OUTPUT_DIR Output directory, default: artifacts/detection-quality-matrix/. + REAL_IOU_THRESHOLD QA IoU threshold, default inherited by the underlying workflow. + +This script never downloads models and never uses fixture detections. It repeats +the existing real-data upload/tile/detection/QA workflow for each matrix row. +EOF +} + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$ROOT" + +BASE_URL="${1:-${GE_INTEL_BASE_URL:-http://localhost:1202}}" +REAL_RASTER_PATH="${2:-${REAL_RASTER_PATH:-}}" +REAL_REFERENCE_VECTOR_PATH="${3:-${REAL_REFERENCE_VECTOR_PATH:-}}" +QUALITY_MODEL_ASSET_IDS="${QUALITY_MODEL_ASSET_IDS:-${REAL_MODEL_ASSET_ID:-__active__}}" +QUALITY_TILE_SIZES="${QUALITY_TILE_SIZES:-640}" +QUALITY_TILE_OVERLAPS="${QUALITY_TILE_OVERLAPS:-64}" +QUALITY_THRESHOLDS="${QUALITY_THRESHOLDS:-0.50 0.25 0.15}" +QUALITY_OUTPUT_DIR="${QUALITY_OUTPUT_DIR:-artifacts/detection-quality-matrix/$(date -u +%Y%m%dT%H%M%SZ)}" + +if [ "${BASE_URL}" = "-h" ] || [ "${BASE_URL}" = "--help" ]; then + usage + exit 0 +fi + +if [ -z "${REAL_RASTER_PATH}" ] || [ -z "${REAL_REFERENCE_VECTOR_PATH}" ]; then + usage + exit 2 +fi + +if ! command -v curl >/dev/null 2>&1; then + echo "curl is required for detection quality matrix verification" >&2 + exit 1 +fi + +if [ -n "${PYTHON_BIN:-}" ]; then + PYTHON_BIN="${PYTHON_BIN}" +else + PYTHON_BIN="" + for candidate in python3 python.exe python; do + if command -v "${candidate}" >/dev/null 2>&1 && "${candidate}" -c "import json, sys" >/dev/null 2>&1; then + PYTHON_BIN="${candidate}" + break + fi + done +fi + +if [ -z "${PYTHON_BIN}" ]; then + echo "A Python interpreter is required for JSON parsing" >&2 + exit 1 +fi + +mkdir -p "${QUALITY_OUTPUT_DIR}" + +model_requests_normalized="$(printf '%s' "${QUALITY_MODEL_ASSET_IDS}" | tr ',' ' ')" +tile_sizes_normalized="$(printf '%s' "${QUALITY_TILE_SIZES}" | tr ',' ' ')" +tile_overlaps_normalized="$(printf '%s' "${QUALITY_TILE_OVERLAPS}" | tr ',' ' ')" +thresholds_normalized="$(printf '%s' "${QUALITY_THRESHOLDS}" | tr ',' ' ')" +matrix_manifest="${QUALITY_OUTPUT_DIR}/quality_matrix_requests.tsv" + +"${PYTHON_BIN}" - \ + "${model_requests_normalized}" \ + "${tile_sizes_normalized}" \ + "${tile_overlaps_normalized}" \ + "${thresholds_normalized}" \ + "${matrix_manifest}" <<'PY' +import re +import sys + +models, tile_sizes, tile_overlaps, thresholds, output_path = sys.argv[1:6] + + +def split_values(raw: str, name: str) -> list[str]: + values = [value.strip() for value in raw.split() if value.strip()] + if not values: + raise SystemExit(f"{name} must contain at least one value") + return values + + +def safe_label(value: str) -> str: + label = re.sub(r"[^A-Za-z0-9_.-]+", "_", value.strip()) + return label.replace(".", "p").replace("-", "_") or "active" + + +model_values = split_values(models, "QUALITY_MODEL_ASSET_IDS") +tile_size_values = split_values(tile_sizes, "QUALITY_TILE_SIZES") +tile_overlap_values = split_values(tile_overlaps, "QUALITY_TILE_OVERLAPS") +threshold_values = split_values(thresholds, "QUALITY_THRESHOLDS") + +for value in tile_size_values: + try: + tile_size = int(value) + except ValueError as exc: + raise SystemExit(f"Invalid tile size: {value}") from exc + if tile_size < 64: + raise SystemExit(f"Tile size must be at least 64: {value}") + +for value in tile_overlap_values: + try: + tile_overlap = int(value) + except ValueError as exc: + raise SystemExit(f"Invalid tile overlap: {value}") from exc + if tile_overlap < 0: + raise SystemExit(f"Tile overlap cannot be negative: {value}") + +for value in threshold_values: + try: + threshold = float(value) + except ValueError as exc: + raise SystemExit(f"Invalid confidence threshold: {value}") from exc + if threshold < 0.0 or threshold > 1.0: + raise SystemExit(f"Confidence threshold must be between 0 and 1: {value}") + +with open(output_path, "w", encoding="utf-8") as handle: + for model in model_values: + for tile_size in tile_size_values: + for overlap in tile_overlap_values: + for threshold in threshold_values: + label = "_".join( + [ + f"model_{safe_label(model)}", + f"tile_{safe_label(tile_size)}", + f"overlap_{safe_label(overlap)}", + f"threshold_{safe_label(threshold)}", + ] + ) + handle.write(f"{model}\t{tile_size}\t{overlap}\t{threshold}\t{label}\n") +PY + +echo "== GeoIntel detection quality matrix ==" +echo "Base URL: ${BASE_URL}" +echo "Raster: ${REAL_RASTER_PATH}" +echo "Reference vector: ${REAL_REFERENCE_VECTOR_PATH}" +echo "Models: ${model_requests_normalized}" +echo "Tile sizes: ${tile_sizes_normalized}" +echo "Tile overlaps: ${tile_overlaps_normalized}" +echo "Thresholds: ${thresholds_normalized}" +echo "Output: ${QUALITY_OUTPUT_DIR}" + +run_index=0 +while IFS=$'\t' read -r model_request tile_size tile_overlap threshold run_label; do + run_index=$((run_index + 1)) + run_log="${QUALITY_OUTPUT_DIR}/${run_label}.log" + quality_response="${QUALITY_OUTPUT_DIR}/${run_label}_quality_checks.json" + run_summary="${QUALITY_OUTPUT_DIR}/${run_label}_summary.json" + model_env="${model_request}" + if [ "${model_request}" = "__active__" ]; then + model_env="" + fi + + echo "-- Matrix run ${run_index}: model=${model_request} tile=${tile_size} overlap=${tile_overlap} threshold=${threshold} --" + if ! REAL_PROJECT_NAME="GeoIntel Detection Quality Matrix ${model_request} tile ${tile_size} overlap ${tile_overlap} threshold ${threshold}" \ + REAL_MODEL_ASSET_ID="${model_env}" \ + REAL_TILE_SIZE="${tile_size}" \ + REAL_TILE_OVERLAP="${tile_overlap}" \ + REAL_CONFIDENCE_THRESHOLD="${threshold}" \ + bash scripts/verify_real_data_detection_qa_workflow.sh "${BASE_URL}" "${REAL_RASTER_PATH}" "${REAL_REFERENCE_VECTOR_PATH}" \ + >"${run_log}" 2>&1; then + echo "Detection quality matrix run failed. Log: ${run_log}" >&2 + tail -n 80 "${run_log}" >&2 || true + exit 1 + fi + + project_id="$(sed -n 's/^Project: //p' "${run_log}" | tail -n 1)" + raster_dataset_id="$(sed -n 's/^Raster dataset: //p' "${run_log}" | tail -n 1)" + reference_dataset_id="$(sed -n 's/^Reference dataset: //p' "${run_log}" | tail -n 1)" + selected_model_asset_id="$(sed -n 's/^Model asset: //p' "${run_log}" | tail -n 1)" + manifest_path="$(sed -n 's/^Manifest: //p' "${run_log}" | tail -n 1)" + analysis_run_id="$(sed -n 's/^Analysis run: //p' "${run_log}" | tail -n 1)" + quality_check_id="$(sed -n 's/^Quality check: //p' "${run_log}" | tail -n 1)" + detection_count="$(sed -n 's/^Detections: //p' "${run_log}" | tail -n 1)" + export_id="$(sed -n 's/^Detection export: //p' "${run_log}" | tail -n 1)" + + if [ -z "${project_id}" ] || [ -z "${analysis_run_id}" ] || [ -z "${quality_check_id}" ]; then + echo "Matrix run did not print project/run/quality ids. Log: ${run_log}" >&2 + exit 1 + fi + + curl -fsS "${BASE_URL%/}/api/v1/projects/${project_id}/quality-checks?limit=200" > "${quality_response}" + + "${PYTHON_BIN}" - \ + "${quality_response}" \ + "${run_summary}" \ + "${model_request}" \ + "${selected_model_asset_id}" \ + "${tile_size}" \ + "${tile_overlap}" \ + "${threshold}" \ + "${project_id}" \ + "${raster_dataset_id}" \ + "${reference_dataset_id}" \ + "${manifest_path}" \ + "${analysis_run_id}" \ + "${quality_check_id}" \ + "${detection_count}" \ + "${export_id}" \ + "${run_log}" <<'PY' +import json +import sys + +( + quality_path, + output_path, + model_request, + selected_model_asset_id, + tile_size, + tile_overlap, + threshold, + project_id, + raster_dataset_id, + reference_dataset_id, + manifest_path, + analysis_run_id, + quality_check_id, + detection_count, + export_id, + run_log, +) = sys.argv[1:17] + +with open(quality_path, "r", encoding="utf-8") as handle: + payload = json.load(handle) +if "data" not in payload: + raise SystemExit("Quality-check list is not a canonical GeoIntel data envelope") +items = payload["data"].get("items") or [] +quality_check = next((item for item in items if str(item.get("id")) == quality_check_id), None) +if quality_check is None: + raise SystemExit(f"Quality check not found in project list: {quality_check_id}") +metrics = { + metric.get("metric_key"): metric.get("metric_value") + for metric in quality_check.get("metrics", []) + if metric.get("metric_key") +} +findings = quality_check.get("findings_json") or {} +summary = { + "model_request": model_request, + "model_asset_id": selected_model_asset_id, + "tile_size": int(tile_size), + "tile_overlap": int(tile_overlap), + "threshold": float(threshold), + "project_id": project_id, + "raster_dataset_id": raster_dataset_id, + "reference_dataset_id": reference_dataset_id, + "manifest_path": manifest_path, + "analysis_run_id": analysis_run_id, + "quality_check_id": quality_check_id, + "detection_count": int(detection_count), + "quality_status": quality_check.get("status"), + "quality_score": quality_check.get("score"), + "precision": metrics.get("precision"), + "recall": metrics.get("recall"), + "f1_score": metrics.get("f1_score", metrics.get("f1")), + "mean_iou": metrics.get("mean_iou"), + "matches": findings.get("matches"), + "false_positives": findings.get("false_positives"), + "false_negatives": findings.get("false_negatives"), + "export_id": export_id, + "run_log": run_log, +} +with open(output_path, "w", encoding="utf-8") as handle: + json.dump(summary, handle, indent=2, sort_keys=True) +print( + "model={model} tile={tile} overlap={overlap} threshold={threshold} detections={detections} " + "score={score} precision={precision} recall={recall} f1={f1} matches={matches} fp={fp} fn={fn}".format( + model=summary["model_asset_id"], + tile=summary["tile_size"], + overlap=summary["tile_overlap"], + threshold=summary["threshold"], + detections=summary["detection_count"], + score=summary["quality_score"], + precision=summary["precision"], + recall=summary["recall"], + f1=summary["f1_score"], + matches=summary["matches"], + fp=summary["false_positives"], + fn=summary["false_negatives"], + ) +) +PY +done < "${matrix_manifest}" + +"${PYTHON_BIN}" - "${QUALITY_OUTPUT_DIR}" "${BASE_URL}" "${matrix_manifest}" <<'PY' +import glob +import json +import os +import sys +from datetime import datetime, timezone + +output_dir, base_url, manifest_path = sys.argv[1:4] +items = [] +for path in sorted(glob.glob(os.path.join(output_dir, "*_summary.json"))): + if path.endswith("quality_matrix_summary.json"): + continue + with open(path, "r", encoding="utf-8") as handle: + items.append(json.load(handle)) + +if not items: + raise SystemExit("No detection quality matrix summaries were produced") + + +def ranked_with(metric: str): + return [item for item in items if item.get(metric) is not None] + + +best_by_score = max(ranked_with("quality_score"), key=lambda item: item["quality_score"], default=None) +best_by_recall = max(ranked_with("recall"), key=lambda item: item["recall"], default=None) +best_by_precision = max(ranked_with("precision"), key=lambda item: item["precision"], default=None) +summary = { + "generated_at": datetime.now(timezone.utc).isoformat(), + "base_url": base_url, + "request_manifest_path": manifest_path, + "run_count": len(items), + "best_by_score": best_by_score, + "best_by_recall": best_by_recall, + "best_by_precision": best_by_precision, + "items": items, +} +summary_path = os.path.join(output_dir, "quality_matrix_summary.json") +with open(summary_path, "w", encoding="utf-8") as handle: + json.dump(summary, handle, indent=2, sort_keys=True) + +print("") +print("Detection quality matrix summary") +print("model\ttile\toverlap\tthreshold\tdetections\tscore\tprecision\trecall\tf1\tmatches\tfp\tfn") +for item in items: + print( + "{model_asset_id}\t{tile_size}\t{tile_overlap}\t{threshold:.2f}\t{detection_count}\t{quality_score}\t{precision}\t{recall}\t{f1_score}\t{matches}\t{false_positives}\t{false_negatives}".format( + **item + ) + ) +print("") +print(f"Summary: {summary_path}") +if best_by_score: + print( + "best_by_score model={model_asset_id} tile={tile_size} overlap={tile_overlap} threshold={threshold:.2f} score={quality_score} f1={f1_score}".format( + **best_by_score + ) + ) +if best_by_recall: + print( + "best_by_recall model={model_asset_id} tile={tile_size} overlap={tile_overlap} threshold={threshold:.2f} recall={recall} detections={detection_count}".format( + **best_by_recall + ) + ) +if best_by_precision: + print( + "best_by_precision model={model_asset_id} tile={tile_size} overlap={tile_overlap} threshold={threshold:.2f} precision={precision} detections={detection_count}".format( + **best_by_precision + ) + ) +PY diff --git a/scripts/run_readiness_check.sh b/scripts/run_readiness_check.sh index 2435d146..2e4227e2 100755 --- a/scripts/run_readiness_check.sh +++ b/scripts/run_readiness_check.sh @@ -57,6 +57,7 @@ bash -n scripts/verify_model_asset_detection_workflow.sh bash -n scripts/verify_real_data_detection_qa_workflow.sh bash -n scripts/run_detection_calibration_sweep.sh bash -n scripts/export_detection_calibration_evidence.sh +bash -n scripts/run_detection_quality_matrix.sh bash -n scripts/verify_workbench_default_state.sh bash -n scripts/verify_workbench_interactions.sh bash -n scripts/verify_gis_runtime.sh