Initial public release

backend/tests/test_sprint12_golden_qa_benchmark.py

@@ -0,0 +1,108 @@
+from __future__ import annotations
+
+import json
+import subprocess
+import sys
+from pathlib import Path
+
+
+ROOT = Path(__file__).resolve().parents[2]
+
+
+def test_golden_qa_expected_metrics_are_documented() -> None:
+    expected_path = ROOT / "fixtures" / "golden" / "expected_qa_metrics.json"
+    expected = json.loads(expected_path.read_text(encoding="utf-8"))
+
+    assert expected["benchmark_id"] == "golden-buildings-partial-match-v1"
+    assert expected["iou_threshold"] == 0.5
+    assert expected["candidate_feature_count"] == 2
+    assert expected["reference_feature_count"] == 2
+    assert expected["matches"] == 1
+    assert expected["false_positive_count"] == 1
+    assert expected["false_negative_count"] == 1
+    assert expected["precision"] == 0.5
+    assert expected["recall"] == 0.5
+    assert expected["f1"] == 0.5
+    assert expected["mean_iou"] > 0.8
+    assert expected["tolerance"] <= 1e-9
+
+
+def test_golden_qa_benchmark_manifest_covers_multiple_regression_scenarios() -> None:
+    manifest_path = ROOT / "fixtures" / "golden" / "golden_qa_benchmarks.json"
+    manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
+    scenario_ids = {scenario["benchmark_id"] for scenario in manifest["scenarios"]}
+
+    assert manifest["version"] == 1
+    assert {
+        "golden-buildings-partial-match-v1",
+        "golden-buildings-perfect-match-v1",
+        "golden-buildings-no-overlap-v1",
+        "golden-buildings-multipolygon-match-v1",
+    }.issubset(scenario_ids)
+
+
+def test_golden_qa_benchmark_command_passes_all_scenarios_and_reports_persistence() -> None:
+    script = ROOT / "scripts" / "run_golden_qa_benchmark.py"
+    result = subprocess.run(
+        [sys.executable, str(script), "--json"],
+        cwd=ROOT,
+        check=True,
+        text=True,
+        capture_output=True,
+    )
+    payload = json.loads(result.stdout)
+
+    assert payload["status"] == "passed"
+    assert payload["scenario_count"] >= 4
+    scenarios = {scenario["benchmark_id"]: scenario for scenario in payload["scenarios"]}
+    partial = scenarios["golden-buildings-partial-match-v1"]
+    perfect = scenarios["golden-buildings-perfect-match-v1"]
+    no_overlap = scenarios["golden-buildings-no-overlap-v1"]
+    multipolygon = scenarios["golden-buildings-multipolygon-match-v1"]
+
+    assert partial["metrics"]["precision"] == 0.5
+    assert partial["metrics"]["recall"] == 0.5
+    assert partial["metrics"]["f1"] == 0.5
+    assert partial["metrics"]["false_positive_count"] == 1
+    assert partial["metrics"]["false_negative_count"] == 1
+    assert perfect["metrics"]["precision"] == 1.0
+    assert perfect["metrics"]["recall"] == 1.0
+    assert perfect["metrics"]["f1"] == 1.0
+    assert perfect["metrics"]["mean_iou"] == 1.0
+    assert no_overlap["metrics"]["precision"] == 0.0
+    assert no_overlap["metrics"]["recall"] == 0.0
+    assert no_overlap["metrics"]["f1"] is None
+    assert no_overlap["metrics"]["mean_iou"] is None
+    assert multipolygon["metrics"]["precision"] == 1.0
+    assert multipolygon["metrics"]["recall"] == 1.0
+    assert multipolygon["metrics"]["f1"] == 1.0
+    assert multipolygon["metrics"]["mean_iou"] == 1.0
+
+    assert payload["persistence"]["quality_check_count"] == payload["scenario_count"]
+    assert payload["persistence"]["metric_count"] == payload["scenario_count"] * 6
+    assert sorted(payload["persistence"]["metric_keys"]) == [
+        "f1",
+        "false_negative_count",
+        "false_positive_count",
+        "mean_iou",
+        "precision",
+        "recall",
+    ]
+
+
+def test_golden_qa_shell_wrapper_is_safe_and_documented() -> None:
+    script = ROOT / "scripts" / "verify_golden_qa_benchmark.sh"
+    content = script.read_text(encoding="utf-8")
+
+    assert "set -euo pipefail" in content
+    assert "run_golden_qa_benchmark.py --json" in content
+    assert "import sys, geoalchemy2" in content
+
+    result = subprocess.run(
+        ["bash", "-n", "scripts/verify_golden_qa_benchmark.sh"],
+        cwd=ROOT,
+        check=True,
+        text=True,
+        capture_output=True,
+    )
+    assert result.returncode == 0