From a28e516e17433d2549cbddd196c683e94a031cb4 Mon Sep 17 00:00:00 2001 From: Codex Date: Wed, 17 Jun 2026 05:56:03 +0200 Subject: [PATCH] Harden golden QA readiness gate --- CHANGELOG.md | 9 ++++ backend/README.md | 10 +++++ backend/tests/test_readiness_gate.py | 8 ++++ .../test_sprint12_golden_qa_benchmark.py | 17 ++++++++ docs/CODEX_EXECUTION_LOG.md | 32 +++++++++++++++ docs/TODO.md | 1 + scripts/README.md | 13 ++++++ scripts/run_readiness_check.sh | 2 + scripts/validate_fixtures.py | 41 ++++++++++++++++--- scripts/verify_golden_qa_benchmark.sh | 23 +++++++++++ 10 files changed, 150 insertions(+), 6 deletions(-) create mode 100644 scripts/verify_golden_qa_benchmark.sh diff --git a/CHANGELOG.md b/CHANGELOG.md index dd7c4e39..20264393 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,15 @@ # Changelog +## Sprint 33 QA/QC benchmark readiness hardening (2026-06-17) + +- Added `scripts/verify_golden_qa_benchmark.sh` as a shell wrapper for the deterministic QA/QC golden benchmark. +- Made `scripts/run_readiness_check.sh` execute the golden QA/QC benchmark and syntax-check the wrapper. +- Hardened fixture validation so `fixtures/golden` GeoJSON files and expected fixture paths are checked. +- Added regression tests to keep the golden benchmark in the readiness gate. +- Updated script/backend docs to document the benchmark wrapper and release gate behavior. +- No API contracts, migrations, product features, live provider fetching or AI behavior were introduced. + ## Sprint 32 Unraid all-in-one runtime (2026-06-17) - Added `docker-compose.unraid.yml` for a single `geointel` container on Unraid. diff --git a/backend/README.md b/backend/README.md index 44555962..9fa501a5 100644 --- a/backend/README.md +++ b/backend/README.md @@ -310,6 +310,12 @@ Machine-readable output: python scripts/run_golden_qa_benchmark.py --json ``` +Shell wrapper used by release-readiness checks: + +```bash +bash scripts/verify_golden_qa_benchmark.sh +``` + The benchmark compares `fixtures/golden/predicted_buildings.geojson` against `fixtures/golden/reference_buildings.geojson` and fails on metric drift. Expected baseline: - precision: `0.5` @@ -321,6 +327,10 @@ The benchmark compares `fixtures/golden/predicted_buildings.geojson` against `fi The command uses existing QA/QC service logic and verifies `QualityCheck`/`Metric` persistence through an in-memory test session. It does not require live providers, AI models, Docker or PostGIS. +`scripts/run_readiness_check.sh` runs this benchmark automatically, so any +change that alters the golden QA/QC metric baseline must update the fixture and +expected metrics deliberately. + ### Demo workflow seed Sprint 15 adds an explicit offline demo workflow seed. It creates or returns a diff --git a/backend/tests/test_readiness_gate.py b/backend/tests/test_readiness_gate.py index f56a0542..81f7f3e5 100644 --- a/backend/tests/test_readiness_gate.py +++ b/backend/tests/test_readiness_gate.py @@ -22,6 +22,14 @@ def test_readiness_gate_checks_demo_export_workflow_script_syntax() -> None: assert "bash -n scripts/verify_demo_export_workflow.sh" in content +def test_readiness_gate_runs_golden_qa_benchmark() -> None: + script = Path(__file__).resolve().parents[2] / "scripts" / "run_readiness_check.sh" + content = script.read_text(encoding="utf-8") + + assert "scripts/run_golden_qa_benchmark.py --json" in content + assert "bash -n scripts/verify_golden_qa_benchmark.sh" in content + + def test_readiness_gate_compiles_demo_cleanup_script() -> None: script = Path(__file__).resolve().parents[2] / "scripts" / "run_readiness_check.sh" content = script.read_text(encoding="utf-8") diff --git a/backend/tests/test_sprint12_golden_qa_benchmark.py b/backend/tests/test_sprint12_golden_qa_benchmark.py index ec25c2cf..aa49a6c0 100644 --- a/backend/tests/test_sprint12_golden_qa_benchmark.py +++ b/backend/tests/test_sprint12_golden_qa_benchmark.py @@ -55,3 +55,20 @@ def test_golden_qa_benchmark_command_passes_and_reports_persistence() -> None: "precision", "recall", ] + + +def test_golden_qa_shell_wrapper_is_safe_and_documented() -> None: + script = ROOT / "scripts" / "verify_golden_qa_benchmark.sh" + content = script.read_text(encoding="utf-8") + + assert "set -euo pipefail" in content + assert "run_golden_qa_benchmark.py --json" in content + + result = subprocess.run( + ["bash", "-n", "scripts/verify_golden_qa_benchmark.sh"], + cwd=ROOT, + check=True, + text=True, + capture_output=True, + ) + assert result.returncode == 0 diff --git a/docs/CODEX_EXECUTION_LOG.md b/docs/CODEX_EXECUTION_LOG.md index 0c2096e7..a50814ad 100644 --- a/docs/CODEX_EXECUTION_LOG.md +++ b/docs/CODEX_EXECUTION_LOG.md @@ -1531,3 +1531,35 @@ Expected Unraid behavior: - A deploy from the repo should no longer leave the final app as a plain Compose-owned container. - The final image/container should avoid Compose metadata labels that can confuse Unraid's Docker page. - The running `geointel` container should expose `net.unraid.docker.managed=dockerman`, web UI metadata and icon metadata immediately after deploy. + +## Sprint 33 QA/QC benchmark readiness hardening (2026-06-17) + +Changed: +- Added `scripts/verify_golden_qa_benchmark.sh` as a shell wrapper for the deterministic QA/QC golden benchmark. +- Made `scripts/run_readiness_check.sh` run `scripts/run_golden_qa_benchmark.py --json` so QA/QC metric drift fails the main release gate. +- Added a readiness syntax check for the golden benchmark wrapper. +- Hardened `scripts/validate_fixtures.py` so `fixtures/golden` GeoJSON files and expected fixture paths are validated alongside the general GeoJSON fixtures. +- Added backend regression tests that keep the golden benchmark wired into readiness. +- Updated script/backend docs, TODO and changelog. + +Tested: +- `python scripts/validate_fixtures.py` +- `python scripts/run_golden_qa_benchmark.py --json` +- `bash -n scripts/verify_golden_qa_benchmark.sh` +- `bash scripts/verify_golden_qa_benchmark.sh` +- `python -m compileall backend/app` +- `cd backend && python -m pytest -W error::DeprecationWarning` +- `cd frontend && npm run typecheck` +- `cd frontend && npm run build` +- `bash scripts/run_readiness_check.sh` +- `cd backend && python -m alembic heads && python -m alembic upgrade head --sql` +- `bash -n scripts/live_migration_smoke.sh` + +Open: +- None for this pass. + +Limitations: +- The benchmark intentionally uses explicit local fixtures only. It does not fetch live GRB/OSM data and does not run AI inference. + +Next recommended pass: +- Continue with broader QA/QC golden demo coverage or frontend export preview decomposition. diff --git a/docs/TODO.md b/docs/TODO.md index 5bcd483f..4033ce35 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -28,6 +28,7 @@ This file now starts with the current implementation status. Older preparation/b - [x] Configured-YOLO optional dependency strategy and local preflight. - [x] Segmentation Lab foundation, persistence, GeoJSON output and QA integration. - [x] QA/QC golden benchmark fixtures and script. +- [x] Run QA/QC golden benchmark from the main readiness gate. - [x] Explicit offline demo workflow seed for project, AOI, fixture datasets and persisted QA metrics. - [x] Project-scoped QA/QC result listing and frontend QA/QC Results panel. - [x] Persisted export foundation for vector/detection/segmentation GeoJSON and project metadata JSON. diff --git a/scripts/README.md b/scripts/README.md index cb31813f..ddf6a444 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -23,6 +23,19 @@ datasets, vector FeatureCollection content, vector feature summary, persisted QA/QC metrics, creates metadata/report/vector GeoJSON exports, lists exports and downloads the JSON/GeoJSON/HTML artifacts through the frontend proxy. +Verify the deterministic QA/QC golden benchmark: + +```bash +bash scripts/verify_golden_qa_benchmark.sh +python scripts/run_golden_qa_benchmark.py --json +``` + +The benchmark uses only explicit local fixtures under `fixtures/golden`, +executes the existing QA/QC matching logic, verifies the expected precision, +recall, F1, mean IoU and false-positive/false-negative counts, and checks that +`QualityCheck` plus `Metric` rows would be persisted. The main readiness gate +runs this benchmark so QA metric drift fails before a release. + Verify a configured local YOLO model without running inference: ```bash diff --git a/scripts/run_readiness_check.sh b/scripts/run_readiness_check.sh index f16a2923..81d5644d 100755 --- a/scripts/run_readiness_check.sh +++ b/scripts/run_readiness_check.sh @@ -35,6 +35,7 @@ ${PYTHON_BIN} scripts/smoke_contracts.py ${PYTHON_BIN} scripts/preimplementation_audit.py ${PYTHON_BIN} scripts/validate_m13_codex_assets.py ${PYTHON_BIN} scripts/validate_m14_launch_assets.py +${PYTHON_BIN} scripts/run_golden_qa_benchmark.py --json >/dev/null ${PYTHON_BIN} -m py_compile scripts/gis_import_smoke.py ${PYTHON_BIN} -m py_compile scripts/seed_demo_workflow.py ${PYTHON_BIN} -m py_compile scripts/yolo_preflight.py @@ -50,4 +51,5 @@ bash -n scripts/live_migration_smoke.sh bash -n scripts/verify_browser_runtime.sh bash -n scripts/verify_demo_export_workflow.sh bash -n scripts/verify_gis_runtime.sh +bash -n scripts/verify_golden_qa_benchmark.sh echo "== Run readiness check passed ==" diff --git a/scripts/validate_fixtures.py b/scripts/validate_fixtures.py index eae44874..2d20b86f 100644 --- a/scripts/validate_fixtures.py +++ b/scripts/validate_fixtures.py @@ -1,9 +1,38 @@ from pathlib import Path import json -ROOT=Path(__file__).resolve().parents[1] -fixture_dir=ROOT/"fixtures"/"geojson" -if not fixture_dir.exists(): raise SystemExit("fixtures/geojson missing") -for path in fixture_dir.glob("*.geojson"): - data=json.loads(path.read_text(encoding="utf-8")) - if data.get("type") != "FeatureCollection": raise SystemExit(f"{path} is not a FeatureCollection") + +ROOT = Path(__file__).resolve().parents[1] + + +def _validate_feature_collection(path: Path) -> None: + data = json.loads(path.read_text(encoding="utf-8")) + if data.get("type") != "FeatureCollection": + raise SystemExit(f"{path} is not a FeatureCollection") + features = data.get("features") + if not isinstance(features, list): + raise SystemExit(f"{path} has no feature list") + for index, feature in enumerate(features): + geometry = feature.get("geometry") if isinstance(feature, dict) else None + if not isinstance(geometry, dict) or not geometry.get("type"): + raise SystemExit(f"{path} feature {index} has no geometry") + + +def _validate_fixture_dir(relative_path: str) -> None: + fixture_dir = ROOT / relative_path + if not fixture_dir.exists(): + raise SystemExit(f"{relative_path} missing") + for path in fixture_dir.glob("*.geojson"): + _validate_feature_collection(path) + + +_validate_fixture_dir("fixtures/geojson") +_validate_fixture_dir("fixtures/golden") + +expected_path = ROOT / "fixtures" / "golden" / "expected_qa_metrics.json" +expected = json.loads(expected_path.read_text(encoding="utf-8")) +for key in ("candidate_fixture", "reference_fixture"): + fixture_path = ROOT / expected[key] + if not fixture_path.exists(): + raise SystemExit(f"Golden QA fixture missing: {fixture_path}") + print("Fixture validation OK") diff --git a/scripts/verify_golden_qa_benchmark.sh b/scripts/verify_golden_qa_benchmark.sh new file mode 100644 index 00000000..b8f84656 --- /dev/null +++ b/scripts/verify_golden_qa_benchmark.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$ROOT" + +if [ -z "${PYTHON_BIN:-}" ]; then + PYTHON_BIN="" + for candidate in python3 python.exe python; do + if command -v "${candidate}" >/dev/null 2>&1 && "${candidate}" -c "import sys" >/dev/null 2>&1; then + PYTHON_BIN="${candidate}" + break + fi + done +fi + +if [ -z "${PYTHON_BIN:-}" ]; then + echo "No usable python interpreter found" >&2 + exit 1 +fi + +echo "== GeoIntel golden QA/QC benchmark ==" +"${PYTHON_BIN}" scripts/run_golden_qa_benchmark.py --json