diff --git a/CHANGELOG.md b/CHANGELOG.md index 98b38d89..a971a1fb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,13 @@ # Changelog +## Sprint 139 Multi-AOI calibration evidence portfolio (2026-07-08) + +- Added `scripts/assemble_detection_calibration_evidence_portfolio.sh` to package multiple AOI calibration summaries and their persisted QA evidence bundles into one model-review portfolio. +- The assembler copies each summary into a sample folder, runs the existing evidence exporter per AOI and writes `calibration_evidence_portfolio.json` plus `calibration_evidence_portfolio.md`. +- Added readiness syntax coverage and a mocked-endpoint regression test for the portfolio convention. +- No backend API, migration, inference, provider fetching, model download, live data mutation or frontend runtime behavior changed. + ## Sprint 138 Browser calibration evidence bundle smoke (2026-07-08) - Added `scripts/smoke_detection_calibration_evidence_bundle.sh` to exercise the browser `detection-calibration-summary.json` -> QA evidence bundle path locally. diff --git a/backend/tests/test_sprint139_multi_aoi_calibration_evidence_portfolio.py b/backend/tests/test_sprint139_multi_aoi_calibration_evidence_portfolio.py new file mode 100644 index 00000000..fc3503ef --- /dev/null +++ b/backend/tests/test_sprint139_multi_aoi_calibration_evidence_portfolio.py @@ -0,0 +1,178 @@ +import json +import os +import subprocess +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[2] + + +def _bash_path(path: Path) -> str: + value = path.as_posix() + if len(value) > 2 and value[1] == ":": + return f"/mnt/{value[0].lower()}{value[2:]}" + return value + + +def _path_from_stdout(stdout: str, label: str) -> Path: + for line in stdout.splitlines(): + if line.startswith(label): + raw_path = line.split(":", 1)[1].strip() + if raw_path.startswith("/mnt/") and len(raw_path) > 6 and raw_path[6] == "/": + drive = raw_path[5].upper() + return Path(f"{drive}:{raw_path[6:]}") + return Path(raw_path) + raise AssertionError(f"Missing {label!r} path in output:\n{stdout}") + + +def test_multi_aoi_calibration_evidence_portfolio_assembles_existing_evidence(tmp_path) -> None: + script_path = ROOT / "scripts" / "assemble_detection_calibration_evidence_portfolio.sh" + readiness_path = ROOT / "scripts" / "run_readiness_check.sh" + readme_path = ROOT / "scripts" / "README.md" + + assert script_path.exists() + assert "bash -n scripts/assemble_detection_calibration_evidence_portfolio.sh" in readiness_path.read_text( + encoding="utf-8" + ) + assert "calibration-evidence-portfolio-manifest.json" in readme_path.read_text(encoding="utf-8") + + summary_a = tmp_path / "geel-summary.json" + summary_b = tmp_path / "mol-summary.json" + summary_a.write_text( + json.dumps( + { + "export_type": "detection_calibration_summary", + "project_id": "project-geel", + "rows": [ + { + "threshold": 0.15, + "quality_check_id": "qc-geel", + "analysis_run_id": "analysis-geel", + "job_id": "job-geel", + "detection_count": 5, + "quality_score": 0.42, + "precision": 0.7, + "recall": 0.3, + "f1_score": 0.42, + } + ], + } + ), + encoding="utf-8", + ) + summary_b.write_text( + json.dumps( + { + "export_type": "detection_calibration_summary", + "project_id": "project-mol", + "rows": [ + { + "threshold": 0.35, + "quality_check_id": "qc-mol", + "analysis_run_id": "analysis-mol", + "job_id": "job-mol", + "detection_count": 3, + "quality_score": 0.6, + "precision": 1.0, + "recall": 0.43, + "f1_score": 0.6, + } + ], + } + ), + encoding="utf-8", + ) + manifest = tmp_path / "calibration-evidence-portfolio-manifest.json" + manifest.write_text( + json.dumps( + { + "portfolio_name": "Kempen building model smoke", + "model_asset_id": "geointel-building-yolov8s-smoke-pt", + "model_sha256": "abc123", + "notes": "Operator comparison notes stay outside application state.", + "samples": [ + { + "sample_slug": "geel", + "aoi_label": "Geel center", + "summary_path": _bash_path(summary_a), + "operator_notes": "Dense urban validation sample.", + }, + { + "sample_slug": "mol", + "aoi_label": "Mol edge", + "summary_path": _bash_path(summary_b), + "operator_notes": "Lower-density validation sample.", + }, + ], + } + ), + encoding="utf-8", + ) + mock_bin = tmp_path / "mock-bin" + mock_bin.mkdir() + mock_curl = mock_bin / "curl" + mock_curl.write_text( + """#!/usr/bin/env bash +set -euo pipefail +url="${@: -1}" +case "$url" in + */api/v1/projects/project-geel/quality-checks/qc-geel/evidence/geojson) + role="match_candidate" + quality_check_id="qc-geel" + ;; + */api/v1/projects/project-mol/quality-checks/qc-mol/evidence/geojson) + role="false_negative" + quality_check_id="qc-mol" + ;; + *) + echo "Unexpected URL: $url" >&2 + exit 22 + ;; +esac +cat <&2 <<'EOF' +Usage: + bash scripts/assemble_detection_calibration_evidence_portfolio.sh [base_url] /path/to/calibration-evidence-portfolio-manifest.json + +Required manifest shape: + { + "portfolio_name": "Kempen building model calibration", + "model_asset_id": "geointel-building-yolov8s-hardneg160r4e50-pt", + "model_sha256": "optional", + "notes": "optional operator notes", + "samples": [ + { + "sample_slug": "geel", + "aoi_label": "Geel center", + "summary_path": "/path/to/detection-calibration-summary.json", + "operator_notes": "optional AOI notes" + } + ] + } + +Optional environment: + CALIBRATION_PORTFOLIO_OUTPUT_DIR Output directory, default: artifacts/detection-calibration-portfolio/. + CALIBRATION_EVIDENCE_MODE all or best, default: all. Forwarded to export_detection_calibration_evidence.sh. + +This is operator evidence tooling only. It does not run inference, create QA +checks, mutate application data, download models or fetch providers. +EOF +} + +if [ "${1:-}" = "-h" ] || [ "${1:-}" = "--help" ]; then + usage + exit 0 +fi + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$ROOT" + +BASE_URL="${1:-${GE_INTEL_BASE_URL:-http://localhost:1202}}" +MANIFEST_PATH="${2:-${CALIBRATION_PORTFOLIO_MANIFEST_PATH:-}}" +CALIBRATION_EVIDENCE_MODE="${CALIBRATION_EVIDENCE_MODE:-all}" +CURL_BIN="${CURL_BIN:-curl}" +timestamp="$(date -u +%Y%m%dT%H%M%SZ)" +OUTPUT_DIR="${CALIBRATION_PORTFOLIO_OUTPUT_DIR:-${ROOT}/artifacts/detection-calibration-portfolio/${timestamp}}" + +if [ -z "$MANIFEST_PATH" ]; then + usage + exit 2 +fi + +case "${CALIBRATION_EVIDENCE_MODE}" in + all|best) ;; + *) + echo "CALIBRATION_EVIDENCE_MODE must be 'all' or 'best'" >&2 + exit 2 + ;; +esac + +if command -v cygpath >/dev/null 2>&1 && printf '%s' "$OUTPUT_DIR" | grep -Eq '^[A-Za-z]:\\'; then + OUTPUT_DIR="$(cygpath -u "$OUTPUT_DIR")" +fi + +if [ -n "${PYTHON_BIN:-}" ]; then + PYTHON_BIN="${PYTHON_BIN}" +else + PYTHON_BIN="" + for candidate in python3 python.exe python; do + if command -v "${candidate}" >/dev/null 2>&1 && "${candidate}" -c "import json, pathlib, shutil, sys" >/dev/null 2>&1; then + PYTHON_BIN="${candidate}" + break + fi + done +fi + +if [ -z "${PYTHON_BIN}" ]; then + echo "A Python interpreter is required for portfolio assembly" >&2 + exit 1 +fi + +mkdir -p "$OUTPUT_DIR" +SAMPLE_REQUESTS="${OUTPUT_DIR}/portfolio_sample_requests.tsv" +PORTFOLIO_META="${OUTPUT_DIR}/portfolio_input_metadata.json" + +"${PYTHON_BIN}" - "$MANIFEST_PATH" "$OUTPUT_DIR" "$SAMPLE_REQUESTS" "$PORTFOLIO_META" <<'PY' +import json +import os +import re +import shutil +import sys +from pathlib import Path + +manifest_path_raw, output_dir_raw, requests_path_raw, metadata_path_raw = sys.argv[1:5] + + +def as_path(raw: str) -> Path: + return Path(raw).expanduser() + + +manifest_path = as_path(manifest_path_raw) +if not manifest_path.is_file(): + raise SystemExit(f"Manifest is not readable: {manifest_path_raw}") + +output_dir = as_path(output_dir_raw) +samples_dir = output_dir / "samples" +samples_dir.mkdir(parents=True, exist_ok=True) + +manifest = json.loads(manifest_path.read_text(encoding="utf-8-sig")) +samples = manifest.get("samples") or [] +if not isinstance(samples, list) or not samples: + raise SystemExit("Portfolio manifest must contain at least one sample") + + +def safe_slug(raw: str) -> str: + slug = re.sub(r"[^A-Za-z0-9_.-]+", "-", str(raw or "").strip()).strip("-._").lower() + if not slug: + raise SystemExit("Every portfolio sample needs a non-empty sample_slug") + return slug + + +request_rows = [] +normalized_samples = [] +seen_slugs = set() +for sample in samples: + if not isinstance(sample, dict): + raise SystemExit("Every portfolio sample must be an object") + sample_slug = safe_slug(sample.get("sample_slug")) + if sample_slug in seen_slugs: + raise SystemExit(f"Duplicate portfolio sample_slug: {sample_slug}") + seen_slugs.add(sample_slug) + summary_path = as_path(str(sample.get("summary_path") or "")) + if not summary_path.is_file(): + raise SystemExit(f"Sample summary_path is not readable for {sample_slug}: {summary_path}") + sample_dir = samples_dir / sample_slug + evidence_dir = sample_dir / "evidence" + sample_dir.mkdir(parents=True, exist_ok=True) + evidence_dir.mkdir(parents=True, exist_ok=True) + copied_summary = sample_dir / summary_path.name + shutil.copyfile(summary_path, copied_summary) + normalized = { + "sample_slug": sample_slug, + "aoi_label": sample.get("aoi_label") or sample_slug, + "operator_notes": sample.get("operator_notes") or "", + "source_summary_path": str(summary_path), + "copied_summary_path": str(copied_summary), + "evidence_dir": str(evidence_dir), + } + normalized_samples.append(normalized) + request_rows.append((sample_slug, str(copied_summary), str(evidence_dir))) + +metadata = { + "portfolio_name": manifest.get("portfolio_name") or "GeoIntel detection calibration evidence portfolio", + "model_asset_id": manifest.get("model_asset_id"), + "model_sha256": manifest.get("model_sha256"), + "notes": manifest.get("notes") or "", + "manifest_path": str(manifest_path), + "samples": normalized_samples, +} +Path(metadata_path_raw).write_text(json.dumps(metadata, indent=2, sort_keys=True), encoding="utf-8") +with Path(requests_path_raw).open("w", encoding="utf-8") as handle: + for row in request_rows: + handle.write("\t".join(row) + "\n") +PY + +echo "== GeoIntel detection calibration evidence portfolio ==" +echo "Base URL: ${BASE_URL}" +echo "Manifest: ${MANIFEST_PATH}" +echo "Mode: ${CALIBRATION_EVIDENCE_MODE}" +echo "Output: ${OUTPUT_DIR}" + +while IFS=$'\t' read -r sample_slug summary_path evidence_dir; do + echo "-- Portfolio sample ${sample_slug} --" + CALIBRATION_EVIDENCE_DIR="${evidence_dir}" \ + CALIBRATION_EVIDENCE_MODE="${CALIBRATION_EVIDENCE_MODE}" \ + CURL_BIN="${CURL_BIN:-curl}" \ + bash scripts/export_detection_calibration_evidence.sh "${BASE_URL}" "${summary_path}" +done < "$SAMPLE_REQUESTS" + +"${PYTHON_BIN}" - "$PORTFOLIO_META" "$OUTPUT_DIR" "$CALIBRATION_EVIDENCE_MODE" <<'PY' +import json +import os +import sys +from datetime import datetime, timezone +from pathlib import Path + +metadata_path, output_dir_raw, mode = sys.argv[1:4] +output_dir = Path(output_dir_raw) +metadata = json.loads(Path(metadata_path).read_text(encoding="utf-8")) + +samples = [] +total_features = 0 +role_counts: dict[str, int] = {} +for sample in metadata["samples"]: + evidence_dir = Path(sample["evidence_dir"]) + evidence_summary_path = evidence_dir / "calibration_evidence_summary.json" + evidence_geojson_path = evidence_dir / "calibration_evidence.geojson" + evidence_review_path = evidence_dir / "calibration_evidence_review.html" + if not evidence_summary_path.is_file(): + raise SystemExit(f"Missing evidence summary for sample {sample['sample_slug']}: {evidence_summary_path}") + evidence_summary = json.loads(evidence_summary_path.read_text(encoding="utf-8")) + feature_count = int(evidence_summary.get("feature_count") or 0) + total_features += feature_count + for role, count in (evidence_summary.get("role_counts") or {}).items(): + role_counts[role] = role_counts.get(role, 0) + int(count) + runs = evidence_summary.get("runs") or [] + best_run = max( + [run for run in runs if run.get("quality_score") is not None], + key=lambda run: run["quality_score"], + default=None, + ) + samples.append( + { + "sample_slug": sample["sample_slug"], + "aoi_label": sample["aoi_label"], + "operator_notes": sample["operator_notes"], + "source_summary_path": sample["source_summary_path"], + "copied_summary_path": sample["copied_summary_path"], + "evidence_summary_path": str(evidence_summary_path), + "evidence_geojson_path": str(evidence_geojson_path), + "evidence_review_path": str(evidence_review_path), + "evidence_feature_count": feature_count, + "role_counts": evidence_summary.get("role_counts") or {}, + "best_run_by_score": best_run, + "runs": runs, + } + ) + + +def sample_score(item: dict) -> float: + best = item.get("best_run_by_score") or {} + value = best.get("quality_score") + return value if isinstance(value, (int, float)) else float("-inf") + + +best_sample = max(samples, key=sample_score, default=None) +portfolio = { + "generated_at": datetime.now(timezone.utc).isoformat(), + "portfolio_name": metadata["portfolio_name"], + "model_asset_id": metadata.get("model_asset_id"), + "model_sha256": metadata.get("model_sha256"), + "notes": metadata.get("notes") or "", + "mode": mode, + "sample_count": len(samples), + "total_evidence_feature_count": total_features, + "role_counts": dict(sorted(role_counts.items())), + "best_sample_by_score": best_sample, + "samples": samples, +} + +portfolio_path = output_dir / "calibration_evidence_portfolio.json" +portfolio_path.write_text(json.dumps(portfolio, indent=2, sort_keys=True), encoding="utf-8") + +markdown_path = output_dir / "calibration_evidence_portfolio.md" +lines = [ + f"# {portfolio['portfolio_name']}", + "", + f"- Generated: {portfolio['generated_at']}", + f"- Model asset: {portfolio.get('model_asset_id') or 'not recorded'}", + f"- Model SHA256: {portfolio.get('model_sha256') or 'not recorded'}", + f"- Mode: {portfolio['mode']}", + f"- Samples: {portfolio['sample_count']}", + f"- Evidence features: {portfolio['total_evidence_feature_count']}", +] +if portfolio.get("notes"): + lines.extend(["", "## Operator notes", "", portfolio["notes"]]) +lines.extend(["", "## Samples", ""]) +for sample in samples: + best = sample.get("best_run_by_score") or {} + lines.extend( + [ + f"### {sample['aoi_label']} (`{sample['sample_slug']}`)", + "", + f"- Evidence features: {sample['evidence_feature_count']}", + f"- Best threshold: {best.get('threshold')}", + f"- Score: {best.get('quality_score')}", + f"- Precision: {best.get('precision')}", + f"- Recall: {best.get('recall')}", + f"- F1: {best.get('f1_score')}", + f"- Review HTML: `{sample['evidence_review_path']}`", + f"- Evidence GeoJSON: `{sample['evidence_geojson_path']}`", + f"- Evidence summary: `{sample['evidence_summary_path']}`", + ] + ) + if sample.get("operator_notes"): + lines.append(f"- Notes: {sample['operator_notes']}") + lines.append("") +markdown_path.write_text("\n".join(lines), encoding="utf-8") + +print("Detection calibration evidence portfolio passed") +print(f"Portfolio JSON: {portfolio_path}") +print(f"Portfolio Markdown: {markdown_path}") +print(f"Samples: {portfolio['sample_count']}") +print(f"Evidence features: {portfolio['total_evidence_feature_count']}") +PY diff --git a/scripts/export_detection_calibration_evidence.sh b/scripts/export_detection_calibration_evidence.sh index 4922b7c5..4da07327 100644 --- a/scripts/export_detection_calibration_evidence.sh +++ b/scripts/export_detection_calibration_evidence.sh @@ -20,6 +20,7 @@ Required input: Optional environment: CALIBRATION_EVIDENCE_MODE all or best, default: all. CALIBRATION_EVIDENCE_DIR Output directory, default: the summary file directory. + CURL_BIN curl executable, default: curl. EOF } @@ -53,7 +54,8 @@ case "${CALIBRATION_EVIDENCE_MODE}" in ;; esac -if ! command -v curl >/dev/null 2>&1; then +CURL_BIN="${CURL_BIN:-curl}" +if ! command -v "${CURL_BIN}" >/dev/null 2>&1; then echo "curl is required for detection calibration evidence export" >&2 exit 1 fi @@ -153,7 +155,7 @@ while IFS=$'\t' read -r threshold project_id quality_check_id; do threshold_label="$(printf '%s' "${threshold}" | tr '.-' 'pm')" response_path="${CALIBRATION_EVIDENCE_DIR}/threshold_${threshold_label}_evidence_response.json" echo "-- Evidence threshold ${threshold}, quality_check_id ${quality_check_id} --" - curl -fsS "${BASE_URL%/}/api/v1/projects/${project_id}/quality-checks/${quality_check_id}/evidence/geojson" > "${response_path}" + "${CURL_BIN}" -fsS "${BASE_URL%/}/api/v1/projects/${project_id}/quality-checks/${quality_check_id}/evidence/geojson" > "${response_path}" done < "${request_manifest}" "${PYTHON_BIN}" - "${CALIBRATION_SUMMARY_PATH}" "${CALIBRATION_EVIDENCE_DIR}" "${CALIBRATION_EVIDENCE_MODE}" <<'PY' diff --git a/scripts/run_readiness_check.sh b/scripts/run_readiness_check.sh index 58eb610c..146f5108 100755 --- a/scripts/run_readiness_check.sh +++ b/scripts/run_readiness_check.sh @@ -61,6 +61,7 @@ bash -n scripts/verify_real_data_detection_qa_workflow.sh bash -n scripts/run_detection_calibration_sweep.sh bash -n scripts/export_detection_calibration_evidence.sh bash -n scripts/smoke_detection_calibration_evidence_bundle.sh +bash -n scripts/assemble_detection_calibration_evidence_portfolio.sh bash -n scripts/run_detection_quality_matrix.sh bash -n scripts/run_multi_sample_detection_quality_matrix.sh bash -n scripts/run_operator_hard_negative_detection_matrix.sh