Add multi-AOI calibration evidence portfolio

2026-07-08 14:10:02 +02:00
parent c42ae23a61
commit 28a49bd0d5
8 changed files with 553 additions and 2 deletions
@@ -0,0 +1,178 @@
+import json
+import os
+import subprocess
+from pathlib import Path
+
+
+ROOT = Path(__file__).resolve().parents[2]
+
+
+def _bash_path(path: Path) -> str:
+    value = path.as_posix()
+    if len(value) > 2 and value[1] == ":":
+        return f"/mnt/{value[0].lower()}{value[2:]}"
+    return value
+
+
+def _path_from_stdout(stdout: str, label: str) -> Path:
+    for line in stdout.splitlines():
+        if line.startswith(label):
+            raw_path = line.split(":", 1)[1].strip()
+            if raw_path.startswith("/mnt/") and len(raw_path) > 6 and raw_path[6] == "/":
+                drive = raw_path[5].upper()
+                return Path(f"{drive}:{raw_path[6:]}")
+            return Path(raw_path)
+    raise AssertionError(f"Missing {label!r} path in output:\n{stdout}")
+
+
+def test_multi_aoi_calibration_evidence_portfolio_assembles_existing_evidence(tmp_path) -> None:
+    script_path = ROOT / "scripts" / "assemble_detection_calibration_evidence_portfolio.sh"
+    readiness_path = ROOT / "scripts" / "run_readiness_check.sh"
+    readme_path = ROOT / "scripts" / "README.md"
+
+    assert script_path.exists()
+    assert "bash -n scripts/assemble_detection_calibration_evidence_portfolio.sh" in readiness_path.read_text(
+        encoding="utf-8"
+    )
+    assert "calibration-evidence-portfolio-manifest.json" in readme_path.read_text(encoding="utf-8")
+
+    summary_a = tmp_path / "geel-summary.json"
+    summary_b = tmp_path / "mol-summary.json"
+    summary_a.write_text(
+        json.dumps(
+            {
+                "export_type": "detection_calibration_summary",
+                "project_id": "project-geel",
+                "rows": [
+                    {
+                        "threshold": 0.15,
+                        "quality_check_id": "qc-geel",
+                        "analysis_run_id": "analysis-geel",
+                        "job_id": "job-geel",
+                        "detection_count": 5,
+                        "quality_score": 0.42,
+                        "precision": 0.7,
+                        "recall": 0.3,
+                        "f1_score": 0.42,
+                    }
+                ],
+            }
+        ),
+        encoding="utf-8",
+    )
+    summary_b.write_text(
+        json.dumps(
+            {
+                "export_type": "detection_calibration_summary",
+                "project_id": "project-mol",
+                "rows": [
+                    {
+                        "threshold": 0.35,
+                        "quality_check_id": "qc-mol",
+                        "analysis_run_id": "analysis-mol",
+                        "job_id": "job-mol",
+                        "detection_count": 3,
+                        "quality_score": 0.6,
+                        "precision": 1.0,
+                        "recall": 0.43,
+                        "f1_score": 0.6,
+                    }
+                ],
+            }
+        ),
+        encoding="utf-8",
+    )
+    manifest = tmp_path / "calibration-evidence-portfolio-manifest.json"
+    manifest.write_text(
+        json.dumps(
+            {
+                "portfolio_name": "Kempen building model smoke",
+                "model_asset_id": "geointel-building-yolov8s-smoke-pt",
+                "model_sha256": "abc123",
+                "notes": "Operator comparison notes stay outside application state.",
+                "samples": [
+                        {
+                            "sample_slug": "geel",
+                            "aoi_label": "Geel center",
+                            "summary_path": _bash_path(summary_a),
+                            "operator_notes": "Dense urban validation sample.",
+                        },
+                        {
+                            "sample_slug": "mol",
+                            "aoi_label": "Mol edge",
+                            "summary_path": _bash_path(summary_b),
+                            "operator_notes": "Lower-density validation sample.",
+                        },
+                ],
+            }
+        ),
+        encoding="utf-8",
+    )
+    mock_bin = tmp_path / "mock-bin"
+    mock_bin.mkdir()
+    mock_curl = mock_bin / "curl"
+    mock_curl.write_text(
+        """#!/usr/bin/env bash
+set -euo pipefail
+url="${@: -1}"
+case "$url" in
+  */api/v1/projects/project-geel/quality-checks/qc-geel/evidence/geojson)
+    role="match_candidate"
+    quality_check_id="qc-geel"
+    ;;
+  */api/v1/projects/project-mol/quality-checks/qc-mol/evidence/geojson)
+    role="false_negative"
+    quality_check_id="qc-mol"
+    ;;
+  *)
+    echo "Unexpected URL: $url" >&2
+    exit 22
+    ;;
+esac
+cat <<JSON
+{"data":{"quality_check_id":"${quality_check_id}","warnings":[],"geojson":{"type":"FeatureCollection","features":[{"type":"Feature","id":"feature-${role}","properties":{"qa_evidence_role":"${role}","feature_id":"${role}-1"},"geometry":{"type":"Polygon","coordinates":[[[4.9,51.1],[4.91,51.1],[4.91,51.11],[4.9,51.11],[4.9,51.1]]]}}]}}}
+JSON
+""",
+        encoding="utf-8",
+        newline="\n",
+    )
+    mock_curl.chmod(0o755)
+
+    env = os.environ.copy()
+    env["PATH"] = f"{_bash_path(mock_bin)}:{env['PATH']}"
+    output_dir = _bash_path(tmp_path / "portfolio-output")
+    manifest_path = _bash_path(manifest)
+    mock_curl_path = _bash_path(mock_curl)
+    result = subprocess.run(
+        [
+            "bash",
+            "-lc",
+            (
+                f"CURL_BIN='{mock_curl_path}' "
+                f"CALIBRATION_PORTFOLIO_OUTPUT_DIR='{output_dir}' "
+                "bash scripts/assemble_detection_calibration_evidence_portfolio.sh "
+                f"http://mock-geointel '{manifest_path}'"
+            ),
+        ],
+        cwd=ROOT,
+        env=env,
+        check=True,
+        text=True,
+        capture_output=True,
+    )
+
+    assert "Detection calibration evidence portfolio passed" in result.stdout
+    portfolio_path = _path_from_stdout(result.stdout, "Portfolio JSON")
+    markdown_path = _path_from_stdout(result.stdout, "Portfolio Markdown")
+    portfolio = json.loads(portfolio_path.read_text(encoding="utf-8"))
+    markdown = markdown_path.read_text(encoding="utf-8")
+
+    assert portfolio["portfolio_name"] == "Kempen building model smoke"
+    assert portfolio["model_asset_id"] == "geointel-building-yolov8s-smoke-pt"
+    assert portfolio["sample_count"] == 2
+    assert portfolio["total_evidence_feature_count"] == 2
+    assert {sample["sample_slug"] for sample in portfolio["samples"]} == {"geel", "mol"}
+    assert portfolio["best_sample_by_score"]["sample_slug"] == "mol"
+    assert "Geel center" in markdown
+    assert "Mol edge" in markdown
+    assert "calibration_evidence_review.html" in markdown