Jens
faeb58ef6d
GeoIntel release gates / Compile, test, contracts and builds (push) Successful in 1m49s
GeoIntel release gates / Python and npm vulnerability policy (push) Successful in 21s
GeoIntel release gates / Production AI image, SBOM and container scan (push) Successful in 5m39s
GeoIntel release gates / Deploy exact gated revision to Unraid (push) Failing after 58m43s
109 lines
4.0 KiB
Python
109 lines
4.0 KiB
Python
from __future__ import annotations
|
|
|
|
import json
|
|
import subprocess
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
|
|
ROOT = Path(__file__).resolve().parents[2]
|
|
|
|
|
|
def test_golden_qa_expected_metrics_are_documented() -> None:
|
|
expected_path = ROOT / "fixtures" / "golden" / "expected_qa_metrics.json"
|
|
expected = json.loads(expected_path.read_text(encoding="utf-8"))
|
|
|
|
assert expected["benchmark_id"] == "golden-buildings-partial-match-v1"
|
|
assert expected["iou_threshold"] == 0.5
|
|
assert expected["candidate_feature_count"] == 2
|
|
assert expected["reference_feature_count"] == 2
|
|
assert expected["matches"] == 1
|
|
assert expected["false_positive_count"] == 1
|
|
assert expected["false_negative_count"] == 1
|
|
assert expected["precision"] == 0.5
|
|
assert expected["recall"] == 0.5
|
|
assert expected["f1"] == 0.5
|
|
assert expected["mean_iou"] > 0.8
|
|
assert expected["tolerance"] <= 1e-9
|
|
|
|
|
|
def test_golden_qa_benchmark_manifest_covers_multiple_regression_scenarios() -> None:
|
|
manifest_path = ROOT / "fixtures" / "golden" / "golden_qa_benchmarks.json"
|
|
manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
|
|
scenario_ids = {scenario["benchmark_id"] for scenario in manifest["scenarios"]}
|
|
|
|
assert manifest["version"] == 1
|
|
assert {
|
|
"golden-buildings-partial-match-v1",
|
|
"golden-buildings-perfect-match-v1",
|
|
"golden-buildings-no-overlap-v1",
|
|
"golden-buildings-multipolygon-match-v1",
|
|
}.issubset(scenario_ids)
|
|
|
|
|
|
def test_golden_qa_benchmark_command_passes_all_scenarios_and_reports_persistence() -> None:
|
|
script = ROOT / "scripts" / "run_golden_qa_benchmark.py"
|
|
result = subprocess.run(
|
|
[sys.executable, str(script), "--json"],
|
|
cwd=ROOT,
|
|
check=True,
|
|
text=True,
|
|
capture_output=True,
|
|
)
|
|
payload = json.loads(result.stdout)
|
|
|
|
assert payload["status"] == "passed"
|
|
assert payload["scenario_count"] >= 4
|
|
scenarios = {scenario["benchmark_id"]: scenario for scenario in payload["scenarios"]}
|
|
partial = scenarios["golden-buildings-partial-match-v1"]
|
|
perfect = scenarios["golden-buildings-perfect-match-v1"]
|
|
no_overlap = scenarios["golden-buildings-no-overlap-v1"]
|
|
multipolygon = scenarios["golden-buildings-multipolygon-match-v1"]
|
|
|
|
assert partial["metrics"]["precision"] == 0.5
|
|
assert partial["metrics"]["recall"] == 0.5
|
|
assert partial["metrics"]["f1"] == 0.5
|
|
assert partial["metrics"]["false_positive_count"] == 1
|
|
assert partial["metrics"]["false_negative_count"] == 1
|
|
assert perfect["metrics"]["precision"] == 1.0
|
|
assert perfect["metrics"]["recall"] == 1.0
|
|
assert perfect["metrics"]["f1"] == 1.0
|
|
assert perfect["metrics"]["mean_iou"] == 1.0
|
|
assert no_overlap["metrics"]["precision"] == 0.0
|
|
assert no_overlap["metrics"]["recall"] == 0.0
|
|
assert no_overlap["metrics"]["f1"] is None
|
|
assert no_overlap["metrics"]["mean_iou"] is None
|
|
assert multipolygon["metrics"]["precision"] == 1.0
|
|
assert multipolygon["metrics"]["recall"] == 1.0
|
|
assert multipolygon["metrics"]["f1"] == 1.0
|
|
assert multipolygon["metrics"]["mean_iou"] == 1.0
|
|
|
|
assert payload["persistence"]["quality_check_count"] == payload["scenario_count"]
|
|
assert payload["persistence"]["metric_count"] == payload["scenario_count"] * 6
|
|
assert sorted(payload["persistence"]["metric_keys"]) == [
|
|
"f1",
|
|
"false_negative_count",
|
|
"false_positive_count",
|
|
"mean_iou",
|
|
"precision",
|
|
"recall",
|
|
]
|
|
|
|
|
|
def test_golden_qa_shell_wrapper_is_safe_and_documented() -> None:
|
|
script = ROOT / "scripts" / "verify_golden_qa_benchmark.sh"
|
|
content = script.read_text(encoding="utf-8")
|
|
|
|
assert "set -euo pipefail" in content
|
|
assert "run_golden_qa_benchmark.py --json" in content
|
|
assert "import sys, geoalchemy2" in content
|
|
|
|
result = subprocess.run(
|
|
["bash", "-n", "scripts/verify_golden_qa_benchmark.sh"],
|
|
cwd=ROOT,
|
|
check=True,
|
|
text=True,
|
|
capture_output=True,
|
|
)
|
|
assert result.returncode == 0
|