diff --git a/CHANGELOG.md b/CHANGELOG.md index 3d4a2f4b..4758406b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,13 @@ # Changelog +## Sprint 125 Detection calibration evidence bundle (2026-07-07) + +- Added `scripts/export_detection_calibration_evidence.sh` to export persisted QA evidence from a detection calibration summary. +- The script writes combined `calibration_evidence.geojson`, `calibration_evidence_summary.json` and a standalone `calibration_evidence_review.html` SVG artifact for matched detections, matched references, false positives and false negatives. +- Added readiness syntax coverage and regression coverage for the evidence bundle contract. +- No inference, model dependency, provider fetching, fake data, API contract or frontend runtime behavior changed. + ## Sprint 124 Detection calibration sweep tooling (2026-07-07) - Added `scripts/run_detection_calibration_sweep.sh` to run the existing real-data detection + QA workflow across multiple configured-YOLO confidence thresholds. diff --git a/backend/README.md b/backend/README.md index 6583560b..97bb9d99 100644 --- a/backend/README.md +++ b/backend/README.md @@ -389,6 +389,19 @@ with detection count, score, precision, recall, F1, mean IoU and false positive/negative counts. It is intended to tune confidence/IoU/model choices, not to add new inference behavior. +To inspect the evidence behind a calibration run, export the persisted QA +evidence bundle: + +```bash +CALIBRATION_SUMMARY_PATH=/mnt/user/appdata/geointel/artifacts/detection-calibration/20260707T002103Z/calibration_summary.json \ +bash scripts/export_detection_calibration_evidence.sh http://192.168.10.150:1202 +``` + +The bundle writes combined QA evidence GeoJSON plus a standalone HTML/SVG review +artifact that separates matched detections, matched references, false positives +and false negatives by role. It reads existing persisted `QualityCheck` evidence +only and does not rerun inference. + ### Run backend ```bash diff --git a/backend/tests/test_sprint125_detection_calibration_evidence_bundle.py b/backend/tests/test_sprint125_detection_calibration_evidence_bundle.py new file mode 100644 index 00000000..14605c21 --- /dev/null +++ b/backend/tests/test_sprint125_detection_calibration_evidence_bundle.py @@ -0,0 +1,30 @@ +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[2] + + +def test_detection_calibration_evidence_bundle_exports_persisted_qa_evidence() -> None: + script_path = ROOT / "scripts" / "export_detection_calibration_evidence.sh" + readiness = (ROOT / "scripts" / "run_readiness_check.sh").read_text(encoding="utf-8") + + assert script_path.exists() + script = script_path.read_text(encoding="utf-8") + + assert "bash -n scripts/export_detection_calibration_evidence.sh" in readiness + assert "CALIBRATION_SUMMARY_PATH" in script + assert "calibration_summary.json" in script + assert "/api/v1/projects/${project_id}/quality-checks/${quality_check_id}/evidence/geojson" in script + assert "Response is not a canonical GeoIntel data envelope" in script + assert "calibration_evidence.geojson" in script + assert "calibration_evidence_summary.json" in script + assert "calibration_evidence_review.html" in script + assert "qa_evidence_role" in script + assert "match_candidate" in script + assert "false_positive" in script + assert "false_negative" in script + assert "best_by_score" in script + assert "&2 <<'EOF' +Usage: + CALIBRATION_SUMMARY_PATH=/path/to/calibration_summary.json \ + bash scripts/export_detection_calibration_evidence.sh [base_url] + +or: + bash scripts/export_detection_calibration_evidence.sh [base_url] /path/to/calibration_summary.json + +Required input: + CALIBRATION_SUMMARY_PATH calibration_summary.json produced by run_detection_calibration_sweep.sh. + +Optional environment: + CALIBRATION_EVIDENCE_MODE all or best, default: all. + CALIBRATION_EVIDENCE_DIR Output directory, default: the summary file directory. +EOF +} + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$ROOT" + +BASE_URL="${1:-${GE_INTEL_BASE_URL:-http://localhost:1202}}" +CALIBRATION_SUMMARY_PATH="${2:-${CALIBRATION_SUMMARY_PATH:-}}" +CALIBRATION_EVIDENCE_MODE="${CALIBRATION_EVIDENCE_MODE:-all}" + +if [ "${BASE_URL}" = "-h" ] || [ "${BASE_URL}" = "--help" ]; then + usage + exit 0 +fi + +if [ -z "${CALIBRATION_SUMMARY_PATH}" ]; then + usage + exit 2 +fi + +if [ ! -f "${CALIBRATION_SUMMARY_PATH}" ]; then + echo "CALIBRATION_SUMMARY_PATH does not point to a readable file: ${CALIBRATION_SUMMARY_PATH}" >&2 + exit 2 +fi + +case "${CALIBRATION_EVIDENCE_MODE}" in + all|best) ;; + *) + echo "CALIBRATION_EVIDENCE_MODE must be 'all' or 'best'" >&2 + exit 2 + ;; +esac + +if ! command -v curl >/dev/null 2>&1; then + echo "curl is required for detection calibration evidence export" >&2 + exit 1 +fi + +if [ -n "${PYTHON_BIN:-}" ]; then + PYTHON_BIN="${PYTHON_BIN}" +else + PYTHON_BIN="" + for candidate in python3 python.exe python; do + if command -v "${candidate}" >/dev/null 2>&1 && "${candidate}" -c "import json, sys" >/dev/null 2>&1; then + PYTHON_BIN="${candidate}" + break + fi + done +fi + +if [ -z "${PYTHON_BIN}" ]; then + echo "A Python interpreter is required for JSON parsing" >&2 + exit 1 +fi + +summary_dir="$(cd "$(dirname "${CALIBRATION_SUMMARY_PATH}")" && pwd)" +CALIBRATION_SUMMARY_PATH="$(cd "$(dirname "${CALIBRATION_SUMMARY_PATH}")" && pwd)/$(basename "${CALIBRATION_SUMMARY_PATH}")" +CALIBRATION_EVIDENCE_DIR="${CALIBRATION_EVIDENCE_DIR:-${summary_dir}}" +mkdir -p "${CALIBRATION_EVIDENCE_DIR}" +CALIBRATION_EVIDENCE_DIR="$(cd "${CALIBRATION_EVIDENCE_DIR}" && pwd)" + +request_manifest="${CALIBRATION_EVIDENCE_DIR}/calibration_evidence_requests.tsv" +"${PYTHON_BIN}" - "${CALIBRATION_SUMMARY_PATH}" "${request_manifest}" "${CALIBRATION_EVIDENCE_MODE}" <<'PY' +import json +import sys + +summary_path, manifest_path, mode = sys.argv[1:4] +with open(summary_path, "r", encoding="utf-8") as handle: + summary = json.load(handle) + +items = summary.get("items") or [] +if mode == "best": + best = summary.get("best_by_score") + if not isinstance(best, dict): + raise SystemExit("calibration_summary.json has no best_by_score object") + items = [best] + +if not items: + raise SystemExit("calibration_summary.json contains no calibration items") + +with open(manifest_path, "w", encoding="utf-8") as handle: + for item in items: + project_id = item.get("project_id") + quality_check_id = item.get("quality_check_id") + threshold = item.get("threshold") + if not project_id or not quality_check_id: + raise SystemExit("Calibration item is missing project_id or quality_check_id") + handle.write(f"{threshold}\t{project_id}\t{quality_check_id}\n") +PY + +echo "== GeoIntel detection calibration evidence export ==" +echo "Base URL: ${BASE_URL}" +echo "Summary: ${CALIBRATION_SUMMARY_PATH}" +echo "Mode: ${CALIBRATION_EVIDENCE_MODE}" +echo "Output: ${CALIBRATION_EVIDENCE_DIR}" + +while IFS=$'\t' read -r threshold project_id quality_check_id; do + threshold_label="$(printf '%s' "${threshold}" | tr '.-' 'pm')" + response_path="${CALIBRATION_EVIDENCE_DIR}/threshold_${threshold_label}_evidence_response.json" + echo "-- Evidence threshold ${threshold}, quality_check_id ${quality_check_id} --" + curl -fsS "${BASE_URL%/}/api/v1/projects/${project_id}/quality-checks/${quality_check_id}/evidence/geojson" > "${response_path}" +done < "${request_manifest}" + +"${PYTHON_BIN}" - "${CALIBRATION_SUMMARY_PATH}" "${CALIBRATION_EVIDENCE_DIR}" "${CALIBRATION_EVIDENCE_MODE}" <<'PY' +import glob +import html +import json +import math +import os +import sys +from collections import Counter +from datetime import datetime, timezone + +summary_path, output_dir, mode = sys.argv[1:4] +with open(summary_path, "r", encoding="utf-8") as handle: + calibration_summary = json.load(handle) + +selected_items = calibration_summary.get("items") or [] +if mode == "best": + selected_items = [calibration_summary["best_by_score"]] +items_by_quality_check = {str(item["quality_check_id"]): item for item in selected_items} + +combined_features = [] +evidence_runs = [] +role_counts = Counter() +warnings = [] + +for response_path in sorted(glob.glob(os.path.join(output_dir, "threshold_*_evidence_response.json"))): + with open(response_path, "r", encoding="utf-8") as handle: + payload = json.load(handle) + if "data" not in payload: + raise SystemExit("Response is not a canonical GeoIntel data envelope") + data = payload["data"] + quality_check_id = str(data.get("quality_check_id")) + item = items_by_quality_check.get(quality_check_id) + if item is None: + continue + threshold = item.get("threshold") + run_warnings = data.get("warnings") or [] + warnings.extend(run_warnings) + feature_count_before = len(combined_features) + for feature in (data.get("geojson") or {}).get("features") or []: + properties = dict(feature.get("properties") or {}) + role = properties.get("qa_evidence_role") or "unknown" + role_counts[role] += 1 + properties.update( + { + "calibration_threshold": threshold, + "calibration_quality_score": item.get("quality_score"), + "calibration_precision": item.get("precision"), + "calibration_recall": item.get("recall"), + "calibration_f1_score": item.get("f1_score"), + "calibration_detection_count": item.get("detection_count"), + } + ) + enriched = dict(feature) + enriched["properties"] = properties + enriched["id"] = f"{threshold}:{feature.get('id')}" + combined_features.append(enriched) + evidence_runs.append( + { + "threshold": threshold, + "project_id": item.get("project_id"), + "analysis_run_id": item.get("analysis_run_id"), + "quality_check_id": quality_check_id, + "detection_count": item.get("detection_count"), + "quality_score": item.get("quality_score"), + "precision": item.get("precision"), + "recall": item.get("recall"), + "f1_score": item.get("f1_score"), + "evidence_feature_count": len(combined_features) - feature_count_before, + "warnings": run_warnings, + } + ) + +combined_geojson = { + "type": "FeatureCollection", + "features": combined_features, +} +combined_path = os.path.join(output_dir, "calibration_evidence.geojson") +with open(combined_path, "w", encoding="utf-8") as handle: + json.dump(combined_geojson, handle, indent=2, sort_keys=True) + +summary = { + "generated_at": datetime.now(timezone.utc).isoformat(), + "mode": mode, + "source_summary_path": summary_path, + "feature_count": len(combined_features), + "role_counts": dict(sorted(role_counts.items())), + "warnings": warnings, + "runs": evidence_runs, + "geojson_path": combined_path, +} +bundle_summary_path = os.path.join(output_dir, "calibration_evidence_summary.json") +with open(bundle_summary_path, "w", encoding="utf-8") as handle: + json.dump(summary, handle, indent=2, sort_keys=True) + + +def iter_coords(geometry): + if not isinstance(geometry, dict): + return + coords = geometry.get("coordinates") + if geometry.get("type") == "Point" and isinstance(coords, list) and len(coords) >= 2: + yield float(coords[0]), float(coords[1]) + elif isinstance(coords, list): + stack = [coords] + while stack: + item = stack.pop() + if ( + isinstance(item, list) + and len(item) >= 2 + and all(isinstance(value, (int, float)) for value in item[:2]) + ): + yield float(item[0]), float(item[1]) + elif isinstance(item, list): + stack.extend(item) + + +coordinates = [coord for feature in combined_features for coord in iter_coords(feature.get("geometry"))] +if coordinates: + min_x = min(x for x, _ in coordinates) + max_x = max(x for x, _ in coordinates) + min_y = min(y for _, y in coordinates) + max_y = max(y for _, y in coordinates) +else: + min_x = min_y = 0.0 + max_x = max_y = 1.0 +width = max(max_x - min_x, 0.000001) +height = max(max_y - min_y, 0.000001) + + +def project(x, y): + px = ((x - min_x) / width) * 1120 + 40 + py = 760 - (((y - min_y) / height) * 720 + 20) + return px, py + + +def path_for_geometry(geometry): + projected = [project(x, y) for x, y in iter_coords(geometry)] + if not projected: + return "" + return "M " + " L ".join(f"{x:.2f} {y:.2f}" for x, y in projected) + " Z" + + +role_colors = { + "match_candidate": "#0f766e", + "match_reference": "#22c55e", + "false_positive": "#dc2626", + "false_negative": "#2563eb", +} +paths = [] +for feature in combined_features: + properties = feature.get("properties") or {} + role = properties.get("qa_evidence_role", "unknown") + threshold = properties.get("calibration_threshold") + stroke = role_colors.get(role, "#64748b") + path_data = path_for_geometry(feature.get("geometry")) + if not path_data: + continue + title = html.escape(f"threshold={threshold} role={role} feature={properties.get('feature_id')}") + paths.append( + f'' + f"{title}" + ) + +rows = [] +for run in sorted(evidence_runs, key=lambda item: float(item.get("threshold") or 0)): + rows.append( + "" + f"{run.get('threshold')}" + f"{run.get('detection_count')}" + f"{run.get('quality_score')}" + f"{run.get('precision')}" + f"{run.get('recall')}" + f"{run.get('f1_score')}" + f"{run.get('evidence_feature_count')}" + f"{html.escape(str(run.get('quality_check_id')))}" + "" + ) + +role_items = "".join( + f"
  • {html.escape(role)}{count}
    • " + for role, count in sorted(role_counts.items()) +) + +html_path = os.path.join(output_dir, "calibration_evidence_review.html") +with open(html_path, "w", encoding="utf-8") as handle: + handle.write( + """ + + + + +GeoIntel Detection Calibration Evidence + + + +
    +

    Detection calibration evidence review

    +

    Generated from persisted GeoIntel QualityCheck evidence. Coordinates are rendered directly from the evidence GeoJSON for quick inspection.

    +
    +
    +

    Evidence geometry overview

    + +""" + + "\n".join(paths) + + """ + +
    + +
    +
    +

    Calibration runs

    + + + +""" + + "\n".join(rows) + + """ + +
    thresholddetectionsscoreprecisionrecallf1evidencequality check
    +
    +
    + + +""" + ) + +print("Detection calibration evidence export passed") +print(f"Evidence GeoJSON: {combined_path}") +print(f"Evidence summary: {bundle_summary_path}") +print(f"Evidence review: {html_path}") +print(f"Evidence features: {len(combined_features)}") +for role, count in sorted(role_counts.items()): + print(f"{role}: {count}") +PY diff --git a/scripts/run_readiness_check.sh b/scripts/run_readiness_check.sh index a7a7725a..2435d146 100755 --- a/scripts/run_readiness_check.sh +++ b/scripts/run_readiness_check.sh @@ -56,6 +56,7 @@ bash -n scripts/verify_ai_handoff_interactions.sh bash -n scripts/verify_model_asset_detection_workflow.sh bash -n scripts/verify_real_data_detection_qa_workflow.sh bash -n scripts/run_detection_calibration_sweep.sh +bash -n scripts/export_detection_calibration_evidence.sh bash -n scripts/verify_workbench_default_state.sh bash -n scripts/verify_workbench_interactions.sh bash -n scripts/verify_gis_runtime.sh