Harden golden QA readiness gate
This commit is contained in:
@@ -310,6 +310,12 @@ Machine-readable output:
|
||||
python scripts/run_golden_qa_benchmark.py --json
|
||||
```
|
||||
|
||||
Shell wrapper used by release-readiness checks:
|
||||
|
||||
```bash
|
||||
bash scripts/verify_golden_qa_benchmark.sh
|
||||
```
|
||||
|
||||
The benchmark compares `fixtures/golden/predicted_buildings.geojson` against `fixtures/golden/reference_buildings.geojson` and fails on metric drift. Expected baseline:
|
||||
|
||||
- precision: `0.5`
|
||||
@@ -321,6 +327,10 @@ The benchmark compares `fixtures/golden/predicted_buildings.geojson` against `fi
|
||||
|
||||
The command uses existing QA/QC service logic and verifies `QualityCheck`/`Metric` persistence through an in-memory test session. It does not require live providers, AI models, Docker or PostGIS.
|
||||
|
||||
`scripts/run_readiness_check.sh` runs this benchmark automatically, so any
|
||||
change that alters the golden QA/QC metric baseline must update the fixture and
|
||||
expected metrics deliberately.
|
||||
|
||||
### Demo workflow seed
|
||||
|
||||
Sprint 15 adds an explicit offline demo workflow seed. It creates or returns a
|
||||
|
||||
@@ -22,6 +22,14 @@ def test_readiness_gate_checks_demo_export_workflow_script_syntax() -> None:
|
||||
assert "bash -n scripts/verify_demo_export_workflow.sh" in content
|
||||
|
||||
|
||||
def test_readiness_gate_runs_golden_qa_benchmark() -> None:
|
||||
script = Path(__file__).resolve().parents[2] / "scripts" / "run_readiness_check.sh"
|
||||
content = script.read_text(encoding="utf-8")
|
||||
|
||||
assert "scripts/run_golden_qa_benchmark.py --json" in content
|
||||
assert "bash -n scripts/verify_golden_qa_benchmark.sh" in content
|
||||
|
||||
|
||||
def test_readiness_gate_compiles_demo_cleanup_script() -> None:
|
||||
script = Path(__file__).resolve().parents[2] / "scripts" / "run_readiness_check.sh"
|
||||
content = script.read_text(encoding="utf-8")
|
||||
|
||||
@@ -55,3 +55,20 @@ def test_golden_qa_benchmark_command_passes_and_reports_persistence() -> None:
|
||||
"precision",
|
||||
"recall",
|
||||
]
|
||||
|
||||
|
||||
def test_golden_qa_shell_wrapper_is_safe_and_documented() -> None:
|
||||
script = ROOT / "scripts" / "verify_golden_qa_benchmark.sh"
|
||||
content = script.read_text(encoding="utf-8")
|
||||
|
||||
assert "set -euo pipefail" in content
|
||||
assert "run_golden_qa_benchmark.py --json" in content
|
||||
|
||||
result = subprocess.run(
|
||||
["bash", "-n", "scripts/verify_golden_qa_benchmark.sh"],
|
||||
cwd=ROOT,
|
||||
check=True,
|
||||
text=True,
|
||||
capture_output=True,
|
||||
)
|
||||
assert result.returncode == 0
|
||||
|
||||
Reference in New Issue
Block a user