Expand golden QA benchmark scenarios

CHANGELOG.md

@@ -7,6 +7,14 @@
 
 # Changelog
 
+## Sprint 61 Golden QA scenario expansion (2026-06-18)
+
+- Expanded the deterministic QA/QC golden benchmark from one building scenario to four local fixture scenarios: partial match, perfect match, no-overlap and MultiPolygon match.
+- Added `fixtures/golden/golden_qa_benchmarks.json` as the scenario manifest while preserving the original `expected_qa_metrics.json` baseline for existing demo workflow checks.
+- Updated `scripts/run_golden_qa_benchmark.py` to run every scenario, verify metric drift and report aggregate `QualityCheck`/`Metric` persistence expectations.
+- Added regression coverage for the multi-scenario manifest and aggregate benchmark output.
+- No product behavior, API contract, migration, provider fetching or AI model behavior changed.
+
 ## Sprint 49 Workbench shell UI refactor (2026-06-17)
 
 - Replaced the one-page workbench panel stack with a task-based UI shell.

backend/tests/test_sprint12_golden_qa_benchmark.py

@@ -27,7 +27,21 @@ def test_golden_qa_expected_metrics_are_documented() -> None:
     assert expected["tolerance"] <= 1e-9
 
 
-def test_golden_qa_benchmark_command_passes_and_reports_persistence() -> None:
+def test_golden_qa_benchmark_manifest_covers_multiple_regression_scenarios() -> None:
+    manifest_path = ROOT / "fixtures" / "golden" / "golden_qa_benchmarks.json"
+    manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
+    scenario_ids = {scenario["benchmark_id"] for scenario in manifest["scenarios"]}
+
+    assert manifest["version"] == 1
+    assert {
+        "golden-buildings-partial-match-v1",
+        "golden-buildings-perfect-match-v1",
+        "golden-buildings-no-overlap-v1",
+        "golden-buildings-multipolygon-match-v1",
+    }.issubset(scenario_ids)
+
+
+def test_golden_qa_benchmark_command_passes_all_scenarios_and_reports_persistence() -> None:
     script = ROOT / "scripts" / "run_golden_qa_benchmark.py"
     result = subprocess.run(
         [sys.executable, str(script), "--json"],
@@ -39,14 +53,33 @@ def test_golden_qa_benchmark_command_passes_and_reports_persistence() -> None:
     payload = json.loads(result.stdout)
 
     assert payload["status"] == "passed"
-    assert payload["benchmark_id"] == "golden-buildings-partial-match-v1"
+    assert payload["scenario_count"] >= 4
-    assert payload["metrics"]["precision"] == 0.5
+    scenarios = {scenario["benchmark_id"]: scenario for scenario in payload["scenarios"]}
-    assert payload["metrics"]["recall"] == 0.5
+    partial = scenarios["golden-buildings-partial-match-v1"]
-    assert payload["metrics"]["f1"] == 0.5
+    perfect = scenarios["golden-buildings-perfect-match-v1"]
-    assert payload["metrics"]["false_positive_count"] == 1
+    no_overlap = scenarios["golden-buildings-no-overlap-v1"]
-    assert payload["metrics"]["false_negative_count"] == 1
+    multipolygon = scenarios["golden-buildings-multipolygon-match-v1"]
-    assert payload["persistence"]["quality_check_count"] == 1
+
-    assert payload["persistence"]["metric_count"] == 6
+    assert partial["metrics"]["precision"] == 0.5
+    assert partial["metrics"]["recall"] == 0.5
+    assert partial["metrics"]["f1"] == 0.5
+    assert partial["metrics"]["false_positive_count"] == 1
+    assert partial["metrics"]["false_negative_count"] == 1
+    assert perfect["metrics"]["precision"] == 1.0
+    assert perfect["metrics"]["recall"] == 1.0
+    assert perfect["metrics"]["f1"] == 1.0
+    assert perfect["metrics"]["mean_iou"] == 1.0
+    assert no_overlap["metrics"]["precision"] == 0.0
+    assert no_overlap["metrics"]["recall"] == 0.0
+    assert no_overlap["metrics"]["f1"] is None
+    assert no_overlap["metrics"]["mean_iou"] is None
+    assert multipolygon["metrics"]["precision"] == 1.0
+    assert multipolygon["metrics"]["recall"] == 1.0
+    assert multipolygon["metrics"]["f1"] == 1.0
+    assert multipolygon["metrics"]["mean_iou"] == 1.0
+
+    assert payload["persistence"]["quality_check_count"] == payload["scenario_count"]
+    assert payload["persistence"]["metric_count"] == payload["scenario_count"] * 6
     assert sorted(payload["persistence"]["metric_keys"]) == [
         "f1",
         "false_negative_count",
@@ -63,6 +96,7 @@ def test_golden_qa_shell_wrapper_is_safe_and_documented() -> None:
 
     assert "set -euo pipefail" in content
     assert "run_golden_qa_benchmark.py --json" in content
+    assert "import sys, geoalchemy2" in content
 
     result = subprocess.run(
         ["bash", "-n", "scripts/verify_golden_qa_benchmark.sh"],

docs/CODEX_EXECUTION_LOG.md

@@ -2384,3 +2384,26 @@ Limitations:
 
 Next recommended pass:
 - Expand golden datasets beyond the current building QA fixtures.
+
+## Sprint 61 Golden QA scenario expansion (2026-06-18)
+
+Changed:
+- Added `fixtures/golden/golden_qa_benchmarks.json` as the explicit scenario manifest for deterministic QA/QC regression coverage.
+- Added local golden fixture pairs for perfect-match, no-overlap and MultiPolygon building comparisons.
+- Updated `scripts/run_golden_qa_benchmark.py` to execute all manifest scenarios, assert expected metric drift, persist simulated `QualityCheck`/`Metric` rows for each scenario and report aggregate persistence totals.
+- Preserved backward-compatible top-level benchmark fields for scripts that still read the original single-scenario output shape.
+- Updated `scripts/README.md`, `docs/TODO.md` and `CHANGELOG.md`.
+
+Tested:
+- Red step: `cd backend && python -m pytest tests/test_sprint12_golden_qa_benchmark.py -q` failed on missing multi-scenario manifest/output.
+- `cd backend && python -m pytest tests/test_sprint12_golden_qa_benchmark.py -q` (`4 passed`)
+- `python scripts/run_golden_qa_benchmark.py --json`
+
+Open:
+- Run full readiness and deploy Tower after the expanded golden benchmark passes the release gate.
+
+Limitations:
+- These are deterministic local QA/QC fixtures only. They do not introduce new production QA metrics or provider data.
+
+Next recommended pass:
+- Continue with frontend visual polish backlog or add a live golden benchmark smoke only if a running PostGIS environment needs that extra release signal.

docs/TODO.md

@@ -59,7 +59,8 @@ This file now starts with the current implementation status. Older preparation/b
 - [x] Demo workflow orchestration hook decomposition.
 - [x] Final `App.tsx` import/encoding cleanup and size audit.
 - [x] Optional final bootstrap-effect extraction.
-- [ ] Decide next V1 stabilization focus: golden dataset expansion or frontend visual polish backlog.
+- [x] Decide next V1 stabilization focus: golden dataset expansion or frontend visual polish backlog.
+- [x] Expand golden QA/QC benchmark coverage across partial, perfect, no-overlap and MultiPolygon scenarios.
 
 ## Sprint 8 status
 
@@ -330,4 +331,4 @@ This file now starts with the current implementation status. Older preparation/b
 - [x] Add a live dry-run maintenance smoke for demo export cleanup.
 - [x] Add browser screenshot artifact automation for visual regression handoff.
 - [x] Add backend error-envelope audit for expected user-error paths.
-- [ ] Expand golden datasets beyond building QA fixtures.
+- [x] Expand golden datasets beyond the original single building QA fixture pair.

fixtures/golden/golden_qa_benchmarks.json

@@ -0,0 +1,74 @@
+{
+  "version": 1,
+  "description": "Golden QA/QC regression scenarios for persisted vector candidate-vs-reference comparison.",
+  "scenarios": [
+    {
+      "benchmark_id": "golden-buildings-partial-match-v1",
+      "description": "Two reference building polygons and two candidate building polygons: one candidate matches ref-1, one candidate is a false positive, and ref-2 is a false negative.",
+      "candidate_fixture": "fixtures/golden/predicted_buildings.geojson",
+      "reference_fixture": "fixtures/golden/reference_buildings.geojson",
+      "iou_threshold": 0.5,
+      "candidate_feature_count": 2,
+      "reference_feature_count": 2,
+      "matches": 1,
+      "false_positive_count": 1,
+      "false_negative_count": 1,
+      "precision": 0.5,
+      "recall": 0.5,
+      "f1": 0.5,
+      "mean_iou": 0.8339768339761133,
+      "tolerance": 1e-9
+    },
+    {
+      "benchmark_id": "golden-buildings-perfect-match-v1",
+      "description": "One candidate building polygon exactly matches one reference polygon.",
+      "candidate_fixture": "fixtures/golden/predicted_buildings_perfect.geojson",
+      "reference_fixture": "fixtures/golden/reference_buildings_perfect.geojson",
+      "iou_threshold": 0.5,
+      "candidate_feature_count": 1,
+      "reference_feature_count": 1,
+      "matches": 1,
+      "false_positive_count": 0,
+      "false_negative_count": 0,
+      "precision": 1.0,
+      "recall": 1.0,
+      "f1": 1.0,
+      "mean_iou": 1.0,
+      "tolerance": 1e-9
+    },
+    {
+      "benchmark_id": "golden-buildings-no-overlap-v1",
+      "description": "One candidate polygon and one reference polygon do not overlap.",
+      "candidate_fixture": "fixtures/golden/predicted_buildings_no_overlap.geojson",
+      "reference_fixture": "fixtures/golden/reference_buildings_no_overlap.geojson",
+      "iou_threshold": 0.5,
+      "candidate_feature_count": 1,
+      "reference_feature_count": 1,
+      "matches": 0,
+      "false_positive_count": 1,
+      "false_negative_count": 1,
+      "precision": 0.0,
+      "recall": 0.0,
+      "f1": null,
+      "mean_iou": null,
+      "tolerance": 1e-9
+    },
+    {
+      "benchmark_id": "golden-buildings-multipolygon-match-v1",
+      "description": "One candidate MultiPolygon exactly matches one reference MultiPolygon.",
+      "candidate_fixture": "fixtures/golden/predicted_buildings_multipolygon.geojson",
+      "reference_fixture": "fixtures/golden/reference_buildings_multipolygon.geojson",
+      "iou_threshold": 0.5,
+      "candidate_feature_count": 1,
+      "reference_feature_count": 1,
+      "matches": 1,
+      "false_positive_count": 0,
+      "false_negative_count": 0,
+      "precision": 1.0,
+      "recall": 1.0,
+      "f1": 1.0,
+      "mean_iou": 1.0,
+      "tolerance": 1e-9
+    }
+  ]
+}

fixtures/golden/predicted_buildings_multipolygon.geojson

@@ -0,0 +1,38 @@
+{
+  "type": "FeatureCollection",
+  "name": "golden_predicted_buildings_multipolygon",
+  "features": [
+    {
+      "type": "Feature",
+      "id": "pred-multipolygon-1",
+      "properties": {
+        "id": "pred-multipolygon-1",
+        "class": "building",
+        "confidence": 0.91
+      },
+      "geometry": {
+        "type": "MultiPolygon",
+        "coordinates": [
+          [
+            [
+              [4.9900, 51.1600],
+              [4.9903, 51.1600],
+              [4.9903, 51.1603],
+              [4.9900, 51.1603],
+              [4.9900, 51.1600]
+            ]
+          ],
+          [
+            [
+              [4.9910, 51.1610],
+              [4.9913, 51.1610],
+              [4.9913, 51.1613],
+              [4.9910, 51.1613],
+              [4.9910, 51.1610]
+            ]
+          ]
+        ]
+      }
+    }
+  ]
+}

fixtures/golden/predicted_buildings_no_overlap.geojson

@@ -0,0 +1,27 @@
+{
+  "type": "FeatureCollection",
+  "name": "golden_predicted_buildings_no_overlap",
+  "features": [
+    {
+      "type": "Feature",
+      "id": "pred-no-overlap-1",
+      "properties": {
+        "id": "pred-no-overlap-1",
+        "class": "building",
+        "confidence": 0.82
+      },
+      "geometry": {
+        "type": "Polygon",
+        "coordinates": [
+          [
+            [4.9950, 51.1650],
+            [4.9955, 51.1650],
+            [4.9955, 51.1654],
+            [4.9950, 51.1654],
+            [4.9950, 51.1650]
+          ]
+        ]
+      }
+    }
+  ]
+}

fixtures/golden/predicted_buildings_perfect.geojson

@@ -0,0 +1,27 @@
+{
+  "type": "FeatureCollection",
+  "name": "golden_predicted_buildings_perfect",
+  "features": [
+    {
+      "type": "Feature",
+      "id": "pred-perfect-1",
+      "properties": {
+        "id": "pred-perfect-1",
+        "class": "building",
+        "confidence": 0.99
+      },
+      "geometry": {
+        "type": "Polygon",
+        "coordinates": [
+          [
+            [4.9900, 51.1600],
+            [4.9905, 51.1600],
+            [4.9905, 51.1604],
+            [4.9900, 51.1604],
+            [4.9900, 51.1600]
+          ]
+        ]
+      }
+    }
+  ]
+}

fixtures/golden/reference_buildings_multipolygon.geojson

@@ -0,0 +1,38 @@
+{
+  "type": "FeatureCollection",
+  "name": "golden_reference_buildings_multipolygon",
+  "features": [
+    {
+      "type": "Feature",
+      "id": "ref-multipolygon-1",
+      "properties": {
+        "id": "ref-multipolygon-1",
+        "source": "golden_fixture",
+        "class": "building"
+      },
+      "geometry": {
+        "type": "MultiPolygon",
+        "coordinates": [
+          [
+            [
+              [4.9900, 51.1600],
+              [4.9903, 51.1600],
+              [4.9903, 51.1603],
+              [4.9900, 51.1603],
+              [4.9900, 51.1600]
+            ]
+          ],
+          [
+            [
+              [4.9910, 51.1610],
+              [4.9913, 51.1610],
+              [4.9913, 51.1613],
+              [4.9910, 51.1613],
+              [4.9910, 51.1610]
+            ]
+          ]
+        ]
+      }
+    }
+  ]
+}

fixtures/golden/reference_buildings_no_overlap.geojson

@@ -0,0 +1,27 @@
+{
+  "type": "FeatureCollection",
+  "name": "golden_reference_buildings_no_overlap",
+  "features": [
+    {
+      "type": "Feature",
+      "id": "ref-no-overlap-1",
+      "properties": {
+        "id": "ref-no-overlap-1",
+        "source": "golden_fixture",
+        "class": "building"
+      },
+      "geometry": {
+        "type": "Polygon",
+        "coordinates": [
+          [
+            [4.9900, 51.1600],
+            [4.9905, 51.1600],
+            [4.9905, 51.1604],
+            [4.9900, 51.1604],
+            [4.9900, 51.1600]
+          ]
+        ]
+      }
+    }
+  ]
+}

fixtures/golden/reference_buildings_perfect.geojson

@@ -0,0 +1,27 @@
+{
+  "type": "FeatureCollection",
+  "name": "golden_reference_buildings_perfect",
+  "features": [
+    {
+      "type": "Feature",
+      "id": "ref-perfect-1",
+      "properties": {
+        "id": "ref-perfect-1",
+        "source": "golden_fixture",
+        "class": "building"
+      },
+      "geometry": {
+        "type": "Polygon",
+        "coordinates": [
+          [
+            [4.9900, 51.1600],
+            [4.9905, 51.1600],
+            [4.9905, 51.1604],
+            [4.9900, 51.1604],
+            [4.9900, 51.1600]
+          ]
+        ]
+      }
+    }
+  ]
+}

scripts/README.md

@@ -86,8 +86,10 @@ python scripts/run_golden_qa_benchmark.py --json
 The benchmark uses only explicit local fixtures under `fixtures/golden`,
 executes the existing QA/QC matching logic, verifies the expected precision,
 recall, F1, mean IoU and false-positive/false-negative counts, and checks that
-`QualityCheck` plus `Metric` rows would be persisted. The main readiness gate
+`QualityCheck` plus `Metric` rows would be persisted. Scenarios are listed in
-runs this benchmark so QA metric drift fails before a release.
+`fixtures/golden/golden_qa_benchmarks.json` and currently cover partial match,
+perfect match, no-overlap and MultiPolygon building comparisons. The main
+readiness gate runs this benchmark so QA metric drift fails before a release.
 
 Verify a configured local YOLO model without running inference:
 

scripts/run_golden_qa_benchmark.py

@@ -41,8 +41,17 @@ class BenchmarkSession:
         self.refreshes.append(item)
 
 
-def _load_expected() -> dict:
+def _load_manifest() -> dict:
-    return json.loads((ROOT / "fixtures" / "golden" / "expected_qa_metrics.json").read_text(encoding="utf-8"))
+    manifest_path = ROOT / "fixtures" / "golden" / "golden_qa_benchmarks.json"
+    if manifest_path.exists():
+        manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
+        if not isinstance(manifest.get("scenarios"), list) or not manifest["scenarios"]:
+            raise AssertionError("golden_qa_benchmarks.json must define at least one scenario")
+        return manifest
+
+    # Backwards-compatible fallback for older checkouts and local smoke scripts.
+    expected = json.loads((ROOT / "fixtures" / "golden" / "expected_qa_metrics.json").read_text(encoding="utf-8"))
+    return {"version": 0, "scenarios": [expected]}
 
 
 def _dataset(dataset_id, project_id, name: str, path: Path, *, role: str) -> Dataset:
@@ -62,27 +71,39 @@ def _dataset(dataset_id, project_id, name: str, path: Path, *, role: str) -> Dat
     )
 
 
-def _assert_close(label: str, actual: float | int | None, expected: float | int, tolerance: float) -> None:
+def _assert_close(label: str, actual: float | int | None, expected: float | int | None, tolerance: float) -> None:
+    if expected is None:
+        if actual is not None:
+            raise AssertionError(f"{label} drifted: actual={actual}, expected=None")
+        return
     if actual is None:
         raise AssertionError(f"{label} is None, expected {expected}")
     if abs(float(actual) - float(expected)) > tolerance:
         raise AssertionError(f"{label} drifted: actual={actual}, expected={expected}, tolerance={tolerance}")
 
 
-def run_benchmark() -> dict:
+def _run_scenario(expected: dict, session: BenchmarkSession, project_id) -> dict:
-    expected = _load_expected()
+    added_before = len(session.added)
-    project_id = uuid4()
+    commits_before = session.commits
     candidate_dataset_id = uuid4()
     reference_dataset_id = uuid4()
     candidate_path = ROOT / expected["candidate_fixture"]
     reference_path = ROOT / expected["reference_fixture"]
     tolerance = float(expected["tolerance"])
 
-    session = BenchmarkSession(
+    session.datasets[candidate_dataset_id] = _dataset(
-        [
+        candidate_dataset_id,
-            _dataset(candidate_dataset_id, project_id, "golden_predicted_buildings.geojson", candidate_path, role="source"),
+        project_id,
-            _dataset(reference_dataset_id, project_id, "golden_reference_buildings.geojson", reference_path, role="reference"),
+        Path(expected["candidate_fixture"]).name,
-        ]
+        candidate_path,
+        role="source",
+    )
+    session.datasets[reference_dataset_id] = _dataset(
+        reference_dataset_id,
+        project_id,
+        Path(expected["reference_fixture"]).name,
+        reference_path,
+        role="reference",
     )
 
     result = QaService.compare_candidate_with_reference(
@@ -128,18 +149,25 @@ def run_benchmark() -> dict:
         candidate_dataset_id=candidate_dataset_id,
         metrics=metrics,
     )
-    persisted_metrics = [item for item in session.added if isinstance(item, Metric)]
+    scenario_added = session.added[added_before:]
+    persisted_metrics = [item for item in scenario_added if isinstance(item, Metric)]
 
     return {
         "status": "passed",
         "benchmark_id": expected["benchmark_id"],
+        "description": expected.get("description"),
         "metrics": metrics,
+        "result_counts": {
+            "candidate_feature_count": result.candidate_feature_count,
+            "reference_feature_count": result.reference_feature_count,
+            "matches": result.matches,
+        },
         "quality_check_id": str(quality_check.id),
         "persistence": {
-            "quality_check_count": len([item for item in session.added if isinstance(item, QualityCheck)]),
+            "quality_check_count": len([item for item in scenario_added if isinstance(item, QualityCheck)]),
             "metric_count": len(persisted_metrics),
             "metric_keys": [metric.metric_key for metric in persisted_metrics],
-            "commit_count": session.commits,
+            "commit_count": session.commits - commits_before,
         },
         "fixtures": {
             "candidate": expected["candidate_fixture"],
@@ -148,6 +176,33 @@ def run_benchmark() -> dict:
     }
 
 
+def run_benchmark() -> dict:
+    manifest = _load_manifest()
+    project_id = uuid4()
+    session = BenchmarkSession([])
+    scenarios = [_run_scenario(expected, session, project_id) for expected in manifest["scenarios"]]
+    persisted_metrics = [item for item in session.added if isinstance(item, Metric)]
+    aggregate_metric_keys = sorted({metric.metric_key for metric in persisted_metrics})
+    first = scenarios[0]
+
+    return {
+        "status": "passed",
+        "version": manifest.get("version"),
+        "scenario_count": len(scenarios),
+        "scenarios": scenarios,
+        "persistence": {
+            "quality_check_count": len([item for item in session.added if isinstance(item, QualityCheck)]),
+            "metric_count": len(persisted_metrics),
+            "metric_keys": aggregate_metric_keys,
+            "commit_count": session.commits,
+        },
+        # Compatibility fields for scripts that still read the original single-scenario shape.
+        "benchmark_id": first["benchmark_id"],
+        "metrics": first["metrics"],
+        "fixtures": first["fixtures"],
+    }
+
+
 def main() -> int:
     parser = argparse.ArgumentParser(description="Run GeoIntel golden QA/QC benchmark.")
     parser.add_argument("--json", action="store_true", help="Print machine-readable JSON only.")

scripts/verify_golden_qa_benchmark.sh

@@ -7,11 +7,19 @@ cd "$ROOT"
 if [ -z "${PYTHON_BIN:-}" ]; then
   PYTHON_BIN=""
   for candidate in python3 python.exe python; do
-    if command -v "${candidate}" >/dev/null 2>&1 && "${candidate}" -c "import sys" >/dev/null 2>&1; then
+    if command -v "${candidate}" >/dev/null 2>&1 && "${candidate}" -c "import sys, geoalchemy2" >/dev/null 2>&1; then
       PYTHON_BIN="${candidate}"
       break
     fi
   done
+  if [ -z "${PYTHON_BIN}" ]; then
+    for candidate in python3 python.exe python; do
+      if command -v "${candidate}" >/dev/null 2>&1 && "${candidate}" -c "import sys" >/dev/null 2>&1; then
+        PYTHON_BIN="${candidate}"
+        break
+      fi
+    done
+  fi
 fi
 
 if [ -z "${PYTHON_BIN:-}" ]; then