From 0bb2e828b1dce383e0f46f8e672973c01837cd09 Mon Sep 17 00:00:00 2001 From: Codex Date: Wed, 17 Jun 2026 06:02:03 +0200 Subject: [PATCH] Harden demo golden QA smoke --- CHANGELOG.md | 9 ++ backend/app/services/demo_workflow_service.py | 92 ++++++++++++++++--- backend/tests/test_readiness_gate.py | 3 + backend/tests/test_sprint15_demo_workflow.py | 5 + .../test_sprint21_demo_workflow_smoke.py | 3 + docs/CODEX_EXECUTION_LOG.md | 27 ++++++ docs/TODO.md | 1 + scripts/README.md | 5 +- scripts/verify_demo_export_workflow.sh | 40 +++++++- 9 files changed, 165 insertions(+), 20 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 20264393..03b31ad7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,15 @@ # Changelog +## Sprint 34 browser-facing golden QA demo hardening (2026-06-17) + +- Hardened `scripts/verify_demo_export_workflow.sh` so the browser-facing demo/export smoke compares persisted QA/QC metrics against `fixtures/golden/expected_qa_metrics.json`. +- The runtime smoke now verifies QA/QC status, F1 score, precision, recall, mean IoU, false positives, false negatives and match counts from persisted `quality_checks`/`metrics`. +- Corrected the offline demo AOI to cover the golden building fixtures and made existing demo workflows self-heal stale/unsupported QA checks by syncing the AOI and persisting a fresh golden QA result. +- Added regression tests to keep the golden QA baseline wired into the demo/export smoke. +- Updated script documentation for the stricter runtime QA/QC checks. +- No API contracts, migrations, product features, live provider fetching or AI behavior were introduced. + ## Sprint 33 QA/QC benchmark readiness hardening (2026-06-17) - Added `scripts/verify_golden_qa_benchmark.sh` as a shell wrapper for the deterministic QA/QC golden benchmark. diff --git a/backend/app/services/demo_workflow_service.py b/backend/app/services/demo_workflow_service.py index 72a34354..b6a86241 100644 --- a/backend/app/services/demo_workflow_service.py +++ b/backend/app/services/demo_workflow_service.py @@ -24,6 +24,7 @@ class DemoWorkflowService: AREA_NAME = "Demo AOI - Geel buildings" REFERENCE_FILENAME = "demo_reference_buildings.geojson" CANDIDATE_FILENAME = "demo_predicted_buildings.geojson" + EXPECTED_METRICS_FILENAME = "expected_qa_metrics.json" @staticmethod def _repo_root() -> Path: @@ -49,6 +50,28 @@ class DemoWorkflowService: raw = path.read_bytes() return json.loads(raw.decode("utf-8")), raw + @staticmethod + def _load_expected_metrics() -> dict: + payload, _raw = DemoWorkflowService._load_fixture(DemoWorkflowService.EXPECTED_METRICS_FILENAME) + return payload + + @staticmethod + def _demo_area_geometry() -> dict: + return { + "type": "MultiPolygon", + "coordinates": [ + [ + [ + [4.9895, 51.1595], + [4.9930, 51.1595], + [4.9930, 51.1615], + [4.9895, 51.1615], + [4.9895, 51.1595], + ] + ] + ], + } + @staticmethod def _find_existing_project(db: Session) -> Project | None: projects = ( @@ -90,20 +113,7 @@ class DemoWorkflowService: @staticmethod def _create_area(db: Session, project_id: UUID) -> Area: - geometry = { - "type": "MultiPolygon", - "coordinates": [ - [ - [ - [4.30, 51.18], - [4.45, 51.18], - [4.45, 51.33], - [4.30, 51.33], - [4.30, 51.18], - ] - ] - ], - } + geometry = DemoWorkflowService._demo_area_geometry() multipolygon = normalize_to_multipolygon(geometry) area = Area( id=uuid4(), @@ -119,6 +129,19 @@ class DemoWorkflowService: db.refresh(area) return area + @staticmethod + def _sync_demo_area(db: Session, area: Area) -> Area: + multipolygon = normalize_to_multipolygon(DemoWorkflowService._demo_area_geometry()) + area.name = DemoWorkflowService.AREA_NAME + area.geometry = from_shape(multipolygon, srid=4326) + area.original_crs = "EPSG:4326" + area.area_m2 = area_m2(multipolygon) + area.bbox = from_shape(geometry_bbox_polygon(multipolygon), srid=4326) + db.add(area) + db.commit() + db.refresh(area) + return area + @staticmethod def _create_dataset( db: Session, @@ -232,6 +255,38 @@ class DemoWorkflowService: }, ) + @staticmethod + def _quality_check_matches_expected(db: Session, quality_check: QualityCheck | None) -> bool: + if not quality_check or quality_check.status != "ok": + return False + expected = DemoWorkflowService._load_expected_metrics() + tolerance = float(expected.get("tolerance", 1e-9)) + if quality_check.score is None or abs(float(quality_check.score) - float(expected["f1"])) > tolerance: + return False + findings = quality_check.findings_json or {} + if int(findings.get("matches", -1)) != int(expected["matches"]): + return False + if int(findings.get("false_positives", -1)) != int(expected["false_positive_count"]): + return False + if int(findings.get("false_negatives", -1)) != int(expected["false_negative_count"]): + return False + + metrics = db.query(Metric).filter(Metric.quality_check_id == quality_check.id).all() + metric_values = {metric.metric_key: metric.metric_value for metric in metrics} + required = { + "precision": expected["precision"], + "recall": expected["recall"], + "f1": expected["f1"], + "mean_iou": expected["mean_iou"], + "false_positive_count": expected["false_positive_count"], + "false_negative_count": expected["false_negative_count"], + } + for key, expected_value in required.items(): + actual = metric_values.get(key) + if actual is None or abs(float(actual) - float(expected_value)) > tolerance: + return False + return True + @staticmethod def seed(db: Session) -> DemoWorkflowResponse: existing = DemoWorkflowService._find_existing_project(db) @@ -261,6 +316,15 @@ class DemoWorkflowService: .first() ) if area and reference and candidate and quality_check: + if not DemoWorkflowService._quality_check_matches_expected(db, quality_check): + area = DemoWorkflowService._sync_demo_area(db, area) + quality_check = DemoWorkflowService._persist_qa( + db=db, + project_id=existing.id, + candidate_dataset_id=candidate.id, + reference_dataset_id=reference.id, + area_id=area.id, + ) return DemoWorkflowResponse( project_id=existing.id, area_id=area.id, diff --git a/backend/tests/test_readiness_gate.py b/backend/tests/test_readiness_gate.py index 81f7f3e5..cd718007 100644 --- a/backend/tests/test_readiness_gate.py +++ b/backend/tests/test_readiness_gate.py @@ -58,6 +58,9 @@ def test_demo_export_workflow_script_verifies_export_endpoints() -> None: assert "GeoJSON Polygon/MultiPolygon geometry" in content assert "precision" in content assert "false_negative_count" in content + assert "fixtures/golden/expected_qa_metrics.json" in content + assert "QA/QC metric {key} drifted" in content + assert "Seeded QA/QC match count does not match golden baseline" in content assert "/api/v1/exports/metadata" in content assert "/api/v1/exports/report" in content assert "/api/v1/exports/geojson" in content diff --git a/backend/tests/test_sprint15_demo_workflow.py b/backend/tests/test_sprint15_demo_workflow.py index e49fd3a9..3a5b7f49 100644 --- a/backend/tests/test_sprint15_demo_workflow.py +++ b/backend/tests/test_sprint15_demo_workflow.py @@ -55,6 +55,9 @@ def test_demo_workflow_service_uses_explicit_golden_fixtures() -> None: assert DemoWorkflowService.PROJECT_NAME == "GeoIntel Demo - Building QA" assert DemoWorkflowService.REFERENCE_FILENAME == "demo_reference_buildings.geojson" assert DemoWorkflowService.CANDIDATE_FILENAME == "demo_predicted_buildings.geojson" + assert DemoWorkflowService.EXPECTED_METRICS_FILENAME == "expected_qa_metrics.json" + assert DemoWorkflowService._demo_area_geometry()["coordinates"][0][0][0][0] < 4.99 + assert DemoWorkflowService._demo_area_geometry()["coordinates"][0][0][2][0] > 4.992 def test_demo_workflow_service_supports_container_fixture_mount() -> None: @@ -66,6 +69,8 @@ def test_demo_workflow_service_supports_container_fixture_mount() -> None: assert "project = existing" in service assert "if not reference:" in service assert "if not candidate:" in service + assert "_sync_demo_area" in service + assert "_quality_check_matches_expected" in service def test_demo_workflow_prefers_complete_existing_demo_project() -> None: diff --git a/backend/tests/test_sprint21_demo_workflow_smoke.py b/backend/tests/test_sprint21_demo_workflow_smoke.py index 0bba6984..c4e5f513 100644 --- a/backend/tests/test_sprint21_demo_workflow_smoke.py +++ b/backend/tests/test_sprint21_demo_workflow_smoke.py @@ -16,6 +16,9 @@ def test_demo_workflow_browser_smoke_script_checks_connected_v1_state() -> None: assert "/api/v1/projects/${project_id}/datasets/${candidate_dataset_id}/vector/summary" in script assert "GeoJSON Polygon/MultiPolygon geometry" in script assert "Candidate vector summary does not report persisted features" in script + assert "fixtures/golden/expected_qa_metrics.json" in script + assert "Seeded QA/QC score does not match the golden F1 baseline" in script + assert "QA/QC metric {key} drifted" in script def test_frontend_demo_action_loads_candidate_dataset_details_for_map_layer() -> None: diff --git a/docs/CODEX_EXECUTION_LOG.md b/docs/CODEX_EXECUTION_LOG.md index a50814ad..f86390f4 100644 --- a/docs/CODEX_EXECUTION_LOG.md +++ b/docs/CODEX_EXECUTION_LOG.md @@ -1563,3 +1563,30 @@ Limitations: Next recommended pass: - Continue with broader QA/QC golden demo coverage or frontend export preview decomposition. + +## Sprint 34 browser-facing golden QA demo hardening (2026-06-17) + +Changed: +- Hardened `scripts/verify_demo_export_workflow.sh` so the browser-facing demo/export smoke loads `fixtures/golden/expected_qa_metrics.json`. +- The smoke now verifies persisted QA/QC status, F1 score, precision, recall, mean IoU, false positives, false negatives and match counts against the golden baseline. +- Corrected the offline demo AOI to cover the golden fixture geometries instead of an older broad Kempen placeholder outside the fixture coordinates. +- Made existing demo workflows self-heal stale/unsupported QA checks by syncing the demo AOI and persisting a fresh golden QA/QC result. +- Added regression checks in backend tests so the demo/export smoke cannot regress back to key-existence-only QA/QC validation. +- Updated script documentation, TODO and changelog. + +Tested: +- `python -m compileall backend/app` +- `cd backend && python -m pytest tests/test_sprint15_demo_workflow.py tests/test_sprint21_demo_workflow_smoke.py tests/test_readiness_gate.py -q` +- `cd backend && python -m pytest -W error::DeprecationWarning` +- `cd frontend && npm run typecheck` +- `cd frontend && npm run build` +- `bash scripts/run_readiness_check.sh` + +Open: +- Tower redeploy and browser-facing smoke are still pending after this edit. + +Limitations: +- The demo/export smoke remains intentionally fixture-based and idempotent. It does not fetch live providers and does not run AI inference. + +Next recommended pass: +- Run the full release-readiness gate and then rebuild/deploy to Tower for browser-facing verification. diff --git a/docs/TODO.md b/docs/TODO.md index 4033ce35..39dac79e 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -29,6 +29,7 @@ This file now starts with the current implementation status. Older preparation/b - [x] Segmentation Lab foundation, persistence, GeoJSON output and QA integration. - [x] QA/QC golden benchmark fixtures and script. - [x] Run QA/QC golden benchmark from the main readiness gate. +- [x] Verify browser-facing demo QA/QC metrics against the golden baseline. - [x] Explicit offline demo workflow seed for project, AOI, fixture datasets and persisted QA metrics. - [x] Project-scoped QA/QC result listing and frontend QA/QC Results panel. - [x] Persisted export foundation for vector/detection/segmentation GeoJSON and project metadata JSON. diff --git a/scripts/README.md b/scripts/README.md index ddf6a444..47acc3ae 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -21,7 +21,10 @@ The demo/export smoke is intentionally mutating and idempotent: it seeds the offline fixture demo if needed, verifies the project area GeoJSON, fixture datasets, vector FeatureCollection content, vector feature summary, persisted QA/QC metrics, creates metadata/report/vector GeoJSON exports, lists exports -and downloads the JSON/GeoJSON/HTML artifacts through the frontend proxy. +and downloads the JSON/GeoJSON/HTML artifacts through the frontend proxy. The +persisted QA/QC result is compared against `fixtures/golden/expected_qa_metrics.json` +so runtime demo precision, recall, F1, mean IoU and false-positive/negative +counts cannot drift silently. Verify the deterministic QA/QC golden benchmark: diff --git a/scripts/verify_demo_export_workflow.sh b/scripts/verify_demo_export_workflow.sh index 6ddd7261..c2c01cc1 100755 --- a/scripts/verify_demo_export_workflow.sh +++ b/scripts/verify_demo_export_workflow.sh @@ -2,6 +2,7 @@ set -euo pipefail BASE_URL="${1:-http://localhost:1202}" +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" TMP_DIR="$(mktemp -d)" trap 'rm -rf "${TMP_DIR}"' EXIT @@ -146,22 +147,51 @@ if [ "${quality_count}" -lt 1 ]; then echo "Expected at least one persisted QA/QC result after demo workflow" >&2 exit 1 fi -"${PYTHON_BIN}" - "${TMP_DIR}/quality_checks.json" "${quality_check_id}" <<'PY' +"${PYTHON_BIN}" - "${TMP_DIR}/quality_checks.json" "${quality_check_id}" "${ROOT}/fixtures/golden/expected_qa_metrics.json" <<'PY' import json import sys -path, quality_check_id = sys.argv[1], sys.argv[2] +path, quality_check_id, expected_path = sys.argv[1], sys.argv[2], sys.argv[3] with open(path, "r", encoding="utf-8") as handle: payload = json.load(handle) +with open(expected_path, "r", encoding="utf-8") as handle: + expected = json.load(handle) items = payload["data"]["items"] matches = [item for item in items if item["id"] == quality_check_id] if not matches: raise SystemExit("Seeded quality_check_id is not listed in project QA/QC results") -metric_keys = {metric["metric_key"] for metric in matches[0].get("metrics", [])} -required = {"precision", "recall", "f1", "mean_iou", "false_positive_count", "false_negative_count"} -missing = required - metric_keys +quality_check = matches[0] +if quality_check.get("status") != "ok": + raise SystemExit(f"Seeded QA/QC result has unexpected status: {quality_check.get('status')}") +if abs(float(quality_check.get("score")) - float(expected["f1"])) > float(expected["tolerance"]): + raise SystemExit("Seeded QA/QC score does not match the golden F1 baseline") +metrics = {metric["metric_key"]: metric.get("metric_value") for metric in quality_check.get("metrics", [])} +required = { + "precision": expected["precision"], + "recall": expected["recall"], + "f1": expected["f1"], + "mean_iou": expected["mean_iou"], + "false_positive_count": expected["false_positive_count"], + "false_negative_count": expected["false_negative_count"], +} +missing = set(required) - set(metrics) if missing: raise SystemExit(f"QA/QC result is missing metrics: {sorted(missing)}") +for key, expected_value in required.items(): + actual = metrics[key] + if actual is None: + raise SystemExit(f"QA/QC metric {key} is None") + if abs(float(actual) - float(expected_value)) > float(expected["tolerance"]): + raise SystemExit( + f"QA/QC metric {key} drifted: actual={actual}, expected={expected_value}, tolerance={expected['tolerance']}" + ) +findings = quality_check.get("findings_json") or {} +if int(findings.get("matches", -1)) != int(expected["matches"]): + raise SystemExit("Seeded QA/QC match count does not match golden baseline") +if int(findings.get("false_positives", -1)) != int(expected["false_positive_count"]): + raise SystemExit("Seeded QA/QC false-positive count does not match golden baseline") +if int(findings.get("false_negatives", -1)) != int(expected["false_negative_count"]): + raise SystemExit("Seeded QA/QC false-negative count does not match golden baseline") PY curl -fsS -X POST "${BASE_URL%/}/api/v1/exports/metadata" \