75 lines
2.4 KiB
Python
75 lines
2.4 KiB
Python
from __future__ import annotations
|
|
|
|
import json
|
|
import subprocess
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
|
|
ROOT = Path(__file__).resolve().parents[2]
|
|
|
|
|
|
def test_golden_qa_expected_metrics_are_documented() -> None:
|
|
expected_path = ROOT / "fixtures" / "golden" / "expected_qa_metrics.json"
|
|
expected = json.loads(expected_path.read_text(encoding="utf-8"))
|
|
|
|
assert expected["benchmark_id"] == "golden-buildings-partial-match-v1"
|
|
assert expected["iou_threshold"] == 0.5
|
|
assert expected["candidate_feature_count"] == 2
|
|
assert expected["reference_feature_count"] == 2
|
|
assert expected["matches"] == 1
|
|
assert expected["false_positive_count"] == 1
|
|
assert expected["false_negative_count"] == 1
|
|
assert expected["precision"] == 0.5
|
|
assert expected["recall"] == 0.5
|
|
assert expected["f1"] == 0.5
|
|
assert expected["mean_iou"] > 0.8
|
|
assert expected["tolerance"] <= 1e-9
|
|
|
|
|
|
def test_golden_qa_benchmark_command_passes_and_reports_persistence() -> None:
|
|
script = ROOT / "scripts" / "run_golden_qa_benchmark.py"
|
|
result = subprocess.run(
|
|
[sys.executable, str(script), "--json"],
|
|
cwd=ROOT,
|
|
check=True,
|
|
text=True,
|
|
capture_output=True,
|
|
)
|
|
payload = json.loads(result.stdout)
|
|
|
|
assert payload["status"] == "passed"
|
|
assert payload["benchmark_id"] == "golden-buildings-partial-match-v1"
|
|
assert payload["metrics"]["precision"] == 0.5
|
|
assert payload["metrics"]["recall"] == 0.5
|
|
assert payload["metrics"]["f1"] == 0.5
|
|
assert payload["metrics"]["false_positive_count"] == 1
|
|
assert payload["metrics"]["false_negative_count"] == 1
|
|
assert payload["persistence"]["quality_check_count"] == 1
|
|
assert payload["persistence"]["metric_count"] == 6
|
|
assert sorted(payload["persistence"]["metric_keys"]) == [
|
|
"f1",
|
|
"false_negative_count",
|
|
"false_positive_count",
|
|
"mean_iou",
|
|
"precision",
|
|
"recall",
|
|
]
|
|
|
|
|
|
def test_golden_qa_shell_wrapper_is_safe_and_documented() -> None:
|
|
script = ROOT / "scripts" / "verify_golden_qa_benchmark.sh"
|
|
content = script.read_text(encoding="utf-8")
|
|
|
|
assert "set -euo pipefail" in content
|
|
assert "run_golden_qa_benchmark.py --json" in content
|
|
|
|
result = subprocess.run(
|
|
["bash", "-n", "scripts/verify_golden_qa_benchmark.sh"],
|
|
cwd=ROOT,
|
|
check=True,
|
|
text=True,
|
|
capture_output=True,
|
|
)
|
|
assert result.returncode == 0
|