Add browser calibration evidence bundle smoke

2026-07-08 13:57:09 +02:00
parent f9204f3c79
commit 47d3587902
7 changed files with 347 additions and 0 deletions
@@ -448,6 +448,19 @@ Browser Detection Lab calibration summary exports are supported too:
 bash scripts/export_detection_calibration_evidence.sh http://192.168.10.150:1202 ./detection-calibration-summary.json
 ```

+Run the local browser-summary evidence bundle smoke without touching live
+application data:
+
+```bash
+bash scripts/smoke_detection_calibration_evidence_bundle.sh
+```
+
+The smoke creates a temporary Detection Lab-style calibration summary, mocks
+the canonical persisted QA evidence endpoint responses, runs the real evidence
+exporter and verifies that `calibration_evidence.geojson`,
+`calibration_evidence_summary.json` and `calibration_evidence_review.html` are
+written correctly.
+
 The evidence export reads each persisted `quality_check_id`, calls the existing
 QA evidence GeoJSON endpoint, writes `calibration_evidence.geojson`,
 `calibration_evidence_summary.json` and a standalone
@@ -60,6 +60,7 @@ bash -n scripts/verify_model_asset_detection_workflow.sh
 bash -n scripts/verify_real_data_detection_qa_workflow.sh
 bash -n scripts/run_detection_calibration_sweep.sh
 bash -n scripts/export_detection_calibration_evidence.sh
+bash -n scripts/smoke_detection_calibration_evidence_bundle.sh
 bash -n scripts/run_detection_quality_matrix.sh
 bash -n scripts/run_multi_sample_detection_quality_matrix.sh
 bash -n scripts/run_operator_hard_negative_detection_matrix.sh
@@ -0,0 +1,233 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+usage() {
+  cat >&2 <<'EOF'
+Usage:
+  bash scripts/smoke_detection_calibration_evidence_bundle.sh
+
+Optional environment:
+  CALIBRATION_EVIDENCE_SMOKE_DIR   Output directory for the temporary smoke files.
+
+This smoke creates a browser-style detection-calibration-summary.json, serves
+canonical QA evidence envelopes through a temporary mock curl command, runs the
+real export_detection_calibration_evidence.sh script, and validates the emitted
+GeoJSON, summary JSON and HTML review artifacts.
+EOF
+}
+
+if [ "${1:-}" = "-h" ] || [ "${1:-}" = "--help" ]; then
+  usage
+  exit 0
+fi
+
+ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+cd "$ROOT"
+
+if [ -n "${PYTHON_BIN:-}" ]; then
+  PYTHON_BIN="${PYTHON_BIN}"
+else
+  PYTHON_BIN=""
+  for candidate in python3 python.exe python; do
+    if command -v "${candidate}" >/dev/null 2>&1 && "${candidate}" -c "import json, sys" >/dev/null 2>&1; then
+      PYTHON_BIN="${candidate}"
+      break
+    fi
+  done
+fi
+
+if [ -z "${PYTHON_BIN}" ]; then
+  echo "A Python interpreter is required for the calibration evidence smoke" >&2
+  exit 1
+fi
+
+timestamp="$(date -u +%Y%m%dT%H%M%SZ)"
+BASE_DIR="${CALIBRATION_EVIDENCE_SMOKE_DIR:-${ROOT}/artifacts/detection-calibration-smoke/${timestamp}}"
+
+if command -v cygpath >/dev/null 2>&1 && printf '%s' "$BASE_DIR" | grep -Eq '^[A-Za-z]:\\'; then
+  BASE_DIR="$(cygpath -u "$BASE_DIR")"
+fi
+
+SUMMARY_DIR="${BASE_DIR}/summary"
+OUTPUT_DIR="${BASE_DIR}/evidence"
+MOCK_BIN_DIR="${BASE_DIR}/mock-bin"
+mkdir -p "$SUMMARY_DIR" "$OUTPUT_DIR" "$MOCK_BIN_DIR"
+
+SUMMARY_PATH="${SUMMARY_DIR}/detection-calibration-summary.json"
+
+cat > "$SUMMARY_PATH" <<'JSON'
+{
+  "export_type": "detection_calibration_summary",
+  "project_id": "project-smoke",
+  "quality_check_ids": ["qc-low", "qc-high"],
+  "rows": [
+    {
+      "threshold": 0.15,
+      "analysis_run_id": "analysis-low",
+      "job_id": "job-low",
+      "quality_check_id": "qc-low",
+      "detection_count": 3,
+      "quality_score": 0.5,
+      "precision": 0.5,
+      "recall": 0.5,
+      "f1_score": 0.5
+    },
+    {
+      "threshold": 0.35,
+      "analysis_run_id": "analysis-high",
+      "job_id": "job-high",
+      "quality_check_id": "qc-high",
+      "detection_count": 1,
+      "quality_score": 0.67,
+      "precision": 1.0,
+      "recall": 0.5,
+      "f1_score": 0.67
+    }
+  ]
+}
+JSON
+
+cat > "${MOCK_BIN_DIR}/curl" <<'BASH'
+#!/usr/bin/env bash
+set -euo pipefail
+
+url="${@: -1}"
+
+case "$url" in
+  */api/v1/projects/project-smoke/quality-checks/qc-low/evidence/geojson)
+    cat <<'JSON'
+{
+  "data": {
+    "project_id": "project-smoke",
+    "quality_check_id": "qc-low",
+    "warnings": [],
+    "geojson": {
+      "type": "FeatureCollection",
+      "features": [
+        {
+          "type": "Feature",
+          "id": "low-match-candidate",
+          "properties": {
+            "qa_evidence_role": "match_candidate",
+            "feature_id": "candidate-low-1"
+          },
+          "geometry": {
+            "type": "Polygon",
+            "coordinates": [[[4.9000, 51.1000], [4.9010, 51.1000], [4.9010, 51.1010], [4.9000, 51.1010], [4.9000, 51.1000]]]
+          }
+        },
+        {
+          "type": "Feature",
+          "id": "low-false-positive",
+          "properties": {
+            "qa_evidence_role": "false_positive",
+            "feature_id": "candidate-low-2"
+          },
+          "geometry": {
+            "type": "Polygon",
+            "coordinates": [[[4.9020, 51.1000], [4.9030, 51.1000], [4.9030, 51.1010], [4.9020, 51.1010], [4.9020, 51.1000]]]
+          }
+        }
+      ]
+    }
+  }
+}
+JSON
+    ;;
+  */api/v1/projects/project-smoke/quality-checks/qc-high/evidence/geojson)
+    cat <<'JSON'
+{
+  "data": {
+    "project_id": "project-smoke",
+    "quality_check_id": "qc-high",
+    "warnings": [],
+    "geojson": {
+      "type": "FeatureCollection",
+      "features": [
+        {
+          "type": "Feature",
+          "id": "high-match-reference",
+          "properties": {
+            "qa_evidence_role": "match_reference",
+            "feature_id": "reference-high-1"
+          },
+          "geometry": {
+            "type": "Polygon",
+            "coordinates": [[[4.9040, 51.1000], [4.9050, 51.1000], [4.9050, 51.1010], [4.9040, 51.1010], [4.9040, 51.1000]]]
+          }
+        },
+        {
+          "type": "Feature",
+          "id": "high-false-negative",
+          "properties": {
+            "qa_evidence_role": "false_negative",
+            "feature_id": "reference-high-2"
+          },
+          "geometry": {
+            "type": "Polygon",
+            "coordinates": [[[4.9060, 51.1000], [4.9070, 51.1000], [4.9070, 51.1010], [4.9060, 51.1010], [4.9060, 51.1000]]]
+          }
+        }
+      ]
+    }
+  }
+}
+JSON
+    ;;
+  *)
+    echo "Unexpected mocked evidence URL: $url" >&2
+    exit 22
+    ;;
+esac
+BASH
+chmod +x "${MOCK_BIN_DIR}/curl"
+
+echo "== GeoIntel detection calibration evidence smoke =="
+echo "Using mocked canonical evidence endpoint"
+echo "Summary: ${SUMMARY_PATH}"
+echo "Output: ${OUTPUT_DIR}"
+
+PATH="${MOCK_BIN_DIR}:${PATH}" \
+CALIBRATION_EVIDENCE_DIR="$OUTPUT_DIR" \
+bash scripts/export_detection_calibration_evidence.sh http://mock-geointel "$SUMMARY_PATH"
+
+"${PYTHON_BIN}" - "$OUTPUT_DIR" <<'PY'
+import json
+import os
+import sys
+
+output_dir = sys.argv[1]
+summary_path = os.path.join(output_dir, "calibration_evidence_summary.json")
+geojson_path = os.path.join(output_dir, "calibration_evidence.geojson")
+html_path = os.path.join(output_dir, "calibration_evidence_review.html")
+
+for path in (summary_path, geojson_path, html_path):
+    if not os.path.isfile(path):
+        raise SystemExit(f"Expected evidence artifact is missing: {path}")
+
+with open(summary_path, "r", encoding="utf-8") as handle:
+    summary = json.load(handle)
+with open(geojson_path, "r", encoding="utf-8") as handle:
+    geojson = json.load(handle)
+with open(html_path, "r", encoding="utf-8") as handle:
+    html = handle.read()
+
+expected_roles = {
+    "match_candidate": 1,
+    "match_reference": 1,
+    "false_positive": 1,
+    "false_negative": 1,
+}
+if summary.get("feature_count") != 4:
+    raise SystemExit(f"Expected 4 evidence features, got {summary.get('feature_count')}")
+if summary.get("role_counts") != expected_roles:
+    raise SystemExit(f"Unexpected role counts: {summary.get('role_counts')}")
+if len(summary.get("runs") or []) != 2:
+    raise SystemExit(f"Expected 2 calibration runs, got {len(summary.get('runs') or [])}")
+if len(geojson.get("features") or []) != 4:
+    raise SystemExit(f"Expected 4 combined GeoJSON features, got {len(geojson.get('features') or [])}")
+if "Detection calibration evidence review" not in html:
+    raise SystemExit("Evidence review HTML title is missing")
+PY
+
+echo "Detection calibration evidence smoke passed"