Harden demo golden QA smoke

CHANGELOG.md

@@ -7,6 +7,15 @@
 
 # Changelog
 
+## Sprint 34 browser-facing golden QA demo hardening (2026-06-17)
+
+- Hardened `scripts/verify_demo_export_workflow.sh` so the browser-facing demo/export smoke compares persisted QA/QC metrics against `fixtures/golden/expected_qa_metrics.json`.
+- The runtime smoke now verifies QA/QC status, F1 score, precision, recall, mean IoU, false positives, false negatives and match counts from persisted `quality_checks`/`metrics`.
+- Corrected the offline demo AOI to cover the golden building fixtures and made existing demo workflows self-heal stale/unsupported QA checks by syncing the AOI and persisting a fresh golden QA result.
+- Added regression tests to keep the golden QA baseline wired into the demo/export smoke.
+- Updated script documentation for the stricter runtime QA/QC checks.
+- No API contracts, migrations, product features, live provider fetching or AI behavior were introduced.
+
 ## Sprint 33 QA/QC benchmark readiness hardening (2026-06-17)
 
 - Added `scripts/verify_golden_qa_benchmark.sh` as a shell wrapper for the deterministic QA/QC golden benchmark.

backend/app/services/demo_workflow_service.py

@@ -24,6 +24,7 @@ class DemoWorkflowService:
     AREA_NAME = "Demo AOI - Geel buildings"
     REFERENCE_FILENAME = "demo_reference_buildings.geojson"
     CANDIDATE_FILENAME = "demo_predicted_buildings.geojson"
+    EXPECTED_METRICS_FILENAME = "expected_qa_metrics.json"
 
     @staticmethod
     def _repo_root() -> Path:
@@ -49,6 +50,28 @@ class DemoWorkflowService:
         raw = path.read_bytes()
         return json.loads(raw.decode("utf-8")), raw
 
+    @staticmethod
+    def _load_expected_metrics() -> dict:
+        payload, _raw = DemoWorkflowService._load_fixture(DemoWorkflowService.EXPECTED_METRICS_FILENAME)
+        return payload
+
+    @staticmethod
+    def _demo_area_geometry() -> dict:
+        return {
+            "type": "MultiPolygon",
+            "coordinates": [
+                [
+                    [
+                        [4.9895, 51.1595],
+                        [4.9930, 51.1595],
+                        [4.9930, 51.1615],
+                        [4.9895, 51.1615],
+                        [4.9895, 51.1595],
+                    ]
+                ]
+            ],
+        }
+
     @staticmethod
     def _find_existing_project(db: Session) -> Project | None:
         projects = (
@@ -90,20 +113,7 @@ class DemoWorkflowService:
 
     @staticmethod
     def _create_area(db: Session, project_id: UUID) -> Area:
-        geometry = {
+        geometry = DemoWorkflowService._demo_area_geometry()
-            "type": "MultiPolygon",
-            "coordinates": [
-                [
-                    [
-                        [4.30, 51.18],
-                        [4.45, 51.18],
-                        [4.45, 51.33],
-                        [4.30, 51.33],
-                        [4.30, 51.18],
-                    ]
-                ]
-            ],
-        }
         multipolygon = normalize_to_multipolygon(geometry)
         area = Area(
             id=uuid4(),
@@ -119,6 +129,19 @@ class DemoWorkflowService:
         db.refresh(area)
         return area
 
+    @staticmethod
+    def _sync_demo_area(db: Session, area: Area) -> Area:
+        multipolygon = normalize_to_multipolygon(DemoWorkflowService._demo_area_geometry())
+        area.name = DemoWorkflowService.AREA_NAME
+        area.geometry = from_shape(multipolygon, srid=4326)
+        area.original_crs = "EPSG:4326"
+        area.area_m2 = area_m2(multipolygon)
+        area.bbox = from_shape(geometry_bbox_polygon(multipolygon), srid=4326)
+        db.add(area)
+        db.commit()
+        db.refresh(area)
+        return area
+
     @staticmethod
     def _create_dataset(
         db: Session,
@@ -232,6 +255,38 @@ class DemoWorkflowService:
             },
         )
 
+    @staticmethod
+    def _quality_check_matches_expected(db: Session, quality_check: QualityCheck | None) -> bool:
+        if not quality_check or quality_check.status != "ok":
+            return False
+        expected = DemoWorkflowService._load_expected_metrics()
+        tolerance = float(expected.get("tolerance", 1e-9))
+        if quality_check.score is None or abs(float(quality_check.score) - float(expected["f1"])) > tolerance:
+            return False
+        findings = quality_check.findings_json or {}
+        if int(findings.get("matches", -1)) != int(expected["matches"]):
+            return False
+        if int(findings.get("false_positives", -1)) != int(expected["false_positive_count"]):
+            return False
+        if int(findings.get("false_negatives", -1)) != int(expected["false_negative_count"]):
+            return False
+
+        metrics = db.query(Metric).filter(Metric.quality_check_id == quality_check.id).all()
+        metric_values = {metric.metric_key: metric.metric_value for metric in metrics}
+        required = {
+            "precision": expected["precision"],
+            "recall": expected["recall"],
+            "f1": expected["f1"],
+            "mean_iou": expected["mean_iou"],
+            "false_positive_count": expected["false_positive_count"],
+            "false_negative_count": expected["false_negative_count"],
+        }
+        for key, expected_value in required.items():
+            actual = metric_values.get(key)
+            if actual is None or abs(float(actual) - float(expected_value)) > tolerance:
+                return False
+        return True
+
     @staticmethod
     def seed(db: Session) -> DemoWorkflowResponse:
         existing = DemoWorkflowService._find_existing_project(db)
@@ -261,6 +316,15 @@ class DemoWorkflowService:
                 .first()
             )
             if area and reference and candidate and quality_check:
+                if not DemoWorkflowService._quality_check_matches_expected(db, quality_check):
+                    area = DemoWorkflowService._sync_demo_area(db, area)
+                    quality_check = DemoWorkflowService._persist_qa(
+                        db=db,
+                        project_id=existing.id,
+                        candidate_dataset_id=candidate.id,
+                        reference_dataset_id=reference.id,
+                        area_id=area.id,
+                    )
                 return DemoWorkflowResponse(
                     project_id=existing.id,
                     area_id=area.id,

backend/tests/test_readiness_gate.py

@@ -58,6 +58,9 @@ def test_demo_export_workflow_script_verifies_export_endpoints() -> None:
     assert "GeoJSON Polygon/MultiPolygon geometry" in content
     assert "precision" in content
     assert "false_negative_count" in content
+    assert "fixtures/golden/expected_qa_metrics.json" in content
+    assert "QA/QC metric {key} drifted" in content
+    assert "Seeded QA/QC match count does not match golden baseline" in content
     assert "/api/v1/exports/metadata" in content
     assert "/api/v1/exports/report" in content
     assert "/api/v1/exports/geojson" in content

backend/tests/test_sprint15_demo_workflow.py

@@ -55,6 +55,9 @@ def test_demo_workflow_service_uses_explicit_golden_fixtures() -> None:
     assert DemoWorkflowService.PROJECT_NAME == "GeoIntel Demo - Building QA"
     assert DemoWorkflowService.REFERENCE_FILENAME == "demo_reference_buildings.geojson"
     assert DemoWorkflowService.CANDIDATE_FILENAME == "demo_predicted_buildings.geojson"
+    assert DemoWorkflowService.EXPECTED_METRICS_FILENAME == "expected_qa_metrics.json"
+    assert DemoWorkflowService._demo_area_geometry()["coordinates"][0][0][0][0] < 4.99
+    assert DemoWorkflowService._demo_area_geometry()["coordinates"][0][0][2][0] > 4.992
 
 
 def test_demo_workflow_service_supports_container_fixture_mount() -> None:
@@ -66,6 +69,8 @@ def test_demo_workflow_service_supports_container_fixture_mount() -> None:
     assert "project = existing" in service
     assert "if not reference:" in service
     assert "if not candidate:" in service
+    assert "_sync_demo_area" in service
+    assert "_quality_check_matches_expected" in service
 
 
 def test_demo_workflow_prefers_complete_existing_demo_project() -> None:

backend/tests/test_sprint21_demo_workflow_smoke.py

@@ -16,6 +16,9 @@ def test_demo_workflow_browser_smoke_script_checks_connected_v1_state() -> None:
     assert "/api/v1/projects/${project_id}/datasets/${candidate_dataset_id}/vector/summary" in script
     assert "GeoJSON Polygon/MultiPolygon geometry" in script
     assert "Candidate vector summary does not report persisted features" in script
+    assert "fixtures/golden/expected_qa_metrics.json" in script
+    assert "Seeded QA/QC score does not match the golden F1 baseline" in script
+    assert "QA/QC metric {key} drifted" in script
 
 
 def test_frontend_demo_action_loads_candidate_dataset_details_for_map_layer() -> None:

docs/CODEX_EXECUTION_LOG.md

@@ -1563,3 +1563,30 @@ Limitations:
 
 Next recommended pass:
 - Continue with broader QA/QC golden demo coverage or frontend export preview decomposition.
+
+## Sprint 34 browser-facing golden QA demo hardening (2026-06-17)
+
+Changed:
+- Hardened `scripts/verify_demo_export_workflow.sh` so the browser-facing demo/export smoke loads `fixtures/golden/expected_qa_metrics.json`.
+- The smoke now verifies persisted QA/QC status, F1 score, precision, recall, mean IoU, false positives, false negatives and match counts against the golden baseline.
+- Corrected the offline demo AOI to cover the golden fixture geometries instead of an older broad Kempen placeholder outside the fixture coordinates.
+- Made existing demo workflows self-heal stale/unsupported QA checks by syncing the demo AOI and persisting a fresh golden QA/QC result.
+- Added regression checks in backend tests so the demo/export smoke cannot regress back to key-existence-only QA/QC validation.
+- Updated script documentation, TODO and changelog.
+
+Tested:
+- `python -m compileall backend/app`
+- `cd backend && python -m pytest tests/test_sprint15_demo_workflow.py tests/test_sprint21_demo_workflow_smoke.py tests/test_readiness_gate.py -q`
+- `cd backend && python -m pytest -W error::DeprecationWarning`
+- `cd frontend && npm run typecheck`
+- `cd frontend && npm run build`
+- `bash scripts/run_readiness_check.sh`
+
+Open:
+- Tower redeploy and browser-facing smoke are still pending after this edit.
+
+Limitations:
+- The demo/export smoke remains intentionally fixture-based and idempotent. It does not fetch live providers and does not run AI inference.
+
+Next recommended pass:
+- Run the full release-readiness gate and then rebuild/deploy to Tower for browser-facing verification.

docs/TODO.md

@@ -29,6 +29,7 @@ This file now starts with the current implementation status. Older preparation/b
 - [x] Segmentation Lab foundation, persistence, GeoJSON output and QA integration.
 - [x] QA/QC golden benchmark fixtures and script.
 - [x] Run QA/QC golden benchmark from the main readiness gate.
+- [x] Verify browser-facing demo QA/QC metrics against the golden baseline.
 - [x] Explicit offline demo workflow seed for project, AOI, fixture datasets and persisted QA metrics.
 - [x] Project-scoped QA/QC result listing and frontend QA/QC Results panel.
 - [x] Persisted export foundation for vector/detection/segmentation GeoJSON and project metadata JSON.

scripts/README.md

@@ -21,7 +21,10 @@ The demo/export smoke is intentionally mutating and idempotent: it seeds the
 offline fixture demo if needed, verifies the project area GeoJSON, fixture
 datasets, vector FeatureCollection content, vector feature summary, persisted
 QA/QC metrics, creates metadata/report/vector GeoJSON exports, lists exports
-and downloads the JSON/GeoJSON/HTML artifacts through the frontend proxy.
+and downloads the JSON/GeoJSON/HTML artifacts through the frontend proxy. The
+persisted QA/QC result is compared against `fixtures/golden/expected_qa_metrics.json`
+so runtime demo precision, recall, F1, mean IoU and false-positive/negative
+counts cannot drift silently.
 
 Verify the deterministic QA/QC golden benchmark:
 

scripts/verify_demo_export_workflow.sh

@@ -2,6 +2,7 @@
 set -euo pipefail
 
 BASE_URL="${1:-http://localhost:1202}"
+ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
 TMP_DIR="$(mktemp -d)"
 trap 'rm -rf "${TMP_DIR}"' EXIT
 
@@ -146,22 +147,51 @@ if [ "${quality_count}" -lt 1 ]; then
   echo "Expected at least one persisted QA/QC result after demo workflow" >&2
   exit 1
 fi
-"${PYTHON_BIN}" - "${TMP_DIR}/quality_checks.json" "${quality_check_id}" <<'PY'
+"${PYTHON_BIN}" - "${TMP_DIR}/quality_checks.json" "${quality_check_id}" "${ROOT}/fixtures/golden/expected_qa_metrics.json" <<'PY'
 import json
 import sys
 
-path, quality_check_id = sys.argv[1], sys.argv[2]
+path, quality_check_id, expected_path = sys.argv[1], sys.argv[2], sys.argv[3]
 with open(path, "r", encoding="utf-8") as handle:
     payload = json.load(handle)
+with open(expected_path, "r", encoding="utf-8") as handle:
+    expected = json.load(handle)
 items = payload["data"]["items"]
 matches = [item for item in items if item["id"] == quality_check_id]
 if not matches:
     raise SystemExit("Seeded quality_check_id is not listed in project QA/QC results")
-metric_keys = {metric["metric_key"] for metric in matches[0].get("metrics", [])}
+quality_check = matches[0]
-required = {"precision", "recall", "f1", "mean_iou", "false_positive_count", "false_negative_count"}
+if quality_check.get("status") != "ok":
-missing = required - metric_keys
+    raise SystemExit(f"Seeded QA/QC result has unexpected status: {quality_check.get('status')}")
+if abs(float(quality_check.get("score")) - float(expected["f1"])) > float(expected["tolerance"]):
+    raise SystemExit("Seeded QA/QC score does not match the golden F1 baseline")
+metrics = {metric["metric_key"]: metric.get("metric_value") for metric in quality_check.get("metrics", [])}
+required = {
+    "precision": expected["precision"],
+    "recall": expected["recall"],
+    "f1": expected["f1"],
+    "mean_iou": expected["mean_iou"],
+    "false_positive_count": expected["false_positive_count"],
+    "false_negative_count": expected["false_negative_count"],
+}
+missing = set(required) - set(metrics)
 if missing:
     raise SystemExit(f"QA/QC result is missing metrics: {sorted(missing)}")
+for key, expected_value in required.items():
+    actual = metrics[key]
+    if actual is None:
+        raise SystemExit(f"QA/QC metric {key} is None")
+    if abs(float(actual) - float(expected_value)) > float(expected["tolerance"]):
+        raise SystemExit(
+            f"QA/QC metric {key} drifted: actual={actual}, expected={expected_value}, tolerance={expected['tolerance']}"
+        )
+findings = quality_check.get("findings_json") or {}
+if int(findings.get("matches", -1)) != int(expected["matches"]):
+    raise SystemExit("Seeded QA/QC match count does not match golden baseline")
+if int(findings.get("false_positives", -1)) != int(expected["false_positive_count"]):
+    raise SystemExit("Seeded QA/QC false-positive count does not match golden baseline")
+if int(findings.get("false_negatives", -1)) != int(expected["false_negative_count"]):
+    raise SystemExit("Seeded QA/QC false-negative count does not match golden baseline")
 PY
 
 curl -fsS -X POST "${BASE_URL%/}/api/v1/exports/metadata" \