Expand golden QA benchmark scenarios

scripts/run_golden_qa_benchmark.py

@@ -41,8 +41,17 @@ class BenchmarkSession:
         self.refreshes.append(item)
 
 
-def _load_expected() -> dict:
-    return json.loads((ROOT / "fixtures" / "golden" / "expected_qa_metrics.json").read_text(encoding="utf-8"))
+def _load_manifest() -> dict:
+    manifest_path = ROOT / "fixtures" / "golden" / "golden_qa_benchmarks.json"
+    if manifest_path.exists():
+        manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
+        if not isinstance(manifest.get("scenarios"), list) or not manifest["scenarios"]:
+            raise AssertionError("golden_qa_benchmarks.json must define at least one scenario")
+        return manifest
+
+    # Backwards-compatible fallback for older checkouts and local smoke scripts.
+    expected = json.loads((ROOT / "fixtures" / "golden" / "expected_qa_metrics.json").read_text(encoding="utf-8"))
+    return {"version": 0, "scenarios": [expected]}
 
 
 def _dataset(dataset_id, project_id, name: str, path: Path, *, role: str) -> Dataset:
@@ -62,27 +71,39 @@ def _dataset(dataset_id, project_id, name: str, path: Path, *, role: str) -> Dat
     )
 
 
-def _assert_close(label: str, actual: float | int | None, expected: float | int, tolerance: float) -> None:
+def _assert_close(label: str, actual: float | int | None, expected: float | int | None, tolerance: float) -> None:
+    if expected is None:
+        if actual is not None:
+            raise AssertionError(f"{label} drifted: actual={actual}, expected=None")
+        return
     if actual is None:
         raise AssertionError(f"{label} is None, expected {expected}")
     if abs(float(actual) - float(expected)) > tolerance:
         raise AssertionError(f"{label} drifted: actual={actual}, expected={expected}, tolerance={tolerance}")
 
 
-def run_benchmark() -> dict:
-    expected = _load_expected()
-    project_id = uuid4()
+def _run_scenario(expected: dict, session: BenchmarkSession, project_id) -> dict:
+    added_before = len(session.added)
+    commits_before = session.commits
     candidate_dataset_id = uuid4()
     reference_dataset_id = uuid4()
     candidate_path = ROOT / expected["candidate_fixture"]
     reference_path = ROOT / expected["reference_fixture"]
     tolerance = float(expected["tolerance"])
 
-    session = BenchmarkSession(
-        [
-            _dataset(candidate_dataset_id, project_id, "golden_predicted_buildings.geojson", candidate_path, role="source"),
-            _dataset(reference_dataset_id, project_id, "golden_reference_buildings.geojson", reference_path, role="reference"),
-        ]
+    session.datasets[candidate_dataset_id] = _dataset(
+        candidate_dataset_id,
+        project_id,
+        Path(expected["candidate_fixture"]).name,
+        candidate_path,
+        role="source",
+    )
+    session.datasets[reference_dataset_id] = _dataset(
+        reference_dataset_id,
+        project_id,
+        Path(expected["reference_fixture"]).name,
+        reference_path,
+        role="reference",
     )
 
     result = QaService.compare_candidate_with_reference(
@@ -128,18 +149,25 @@ def run_benchmark() -> dict:
         candidate_dataset_id=candidate_dataset_id,
         metrics=metrics,
     )
-    persisted_metrics = [item for item in session.added if isinstance(item, Metric)]
+    scenario_added = session.added[added_before:]
+    persisted_metrics = [item for item in scenario_added if isinstance(item, Metric)]
 
     return {
         "status": "passed",
         "benchmark_id": expected["benchmark_id"],
+        "description": expected.get("description"),
         "metrics": metrics,
+        "result_counts": {
+            "candidate_feature_count": result.candidate_feature_count,
+            "reference_feature_count": result.reference_feature_count,
+            "matches": result.matches,
+        },
         "quality_check_id": str(quality_check.id),
         "persistence": {
-            "quality_check_count": len([item for item in session.added if isinstance(item, QualityCheck)]),
+            "quality_check_count": len([item for item in scenario_added if isinstance(item, QualityCheck)]),
             "metric_count": len(persisted_metrics),
             "metric_keys": [metric.metric_key for metric in persisted_metrics],
-            "commit_count": session.commits,
+            "commit_count": session.commits - commits_before,
         },
         "fixtures": {
             "candidate": expected["candidate_fixture"],
@@ -148,6 +176,33 @@ def run_benchmark() -> dict:
     }
 
 
+def run_benchmark() -> dict:
+    manifest = _load_manifest()
+    project_id = uuid4()
+    session = BenchmarkSession([])
+    scenarios = [_run_scenario(expected, session, project_id) for expected in manifest["scenarios"]]
+    persisted_metrics = [item for item in session.added if isinstance(item, Metric)]
+    aggregate_metric_keys = sorted({metric.metric_key for metric in persisted_metrics})
+    first = scenarios[0]
+
+    return {
+        "status": "passed",
+        "version": manifest.get("version"),
+        "scenario_count": len(scenarios),
+        "scenarios": scenarios,
+        "persistence": {
+            "quality_check_count": len([item for item in session.added if isinstance(item, QualityCheck)]),
+            "metric_count": len(persisted_metrics),
+            "metric_keys": aggregate_metric_keys,
+            "commit_count": session.commits,
+        },
+        # Compatibility fields for scripts that still read the original single-scenario shape.
+        "benchmark_id": first["benchmark_id"],
+        "metrics": first["metrics"],
+        "fixtures": first["fixtures"],
+    }
+
+
 def main() -> int:
     parser = argparse.ArgumentParser(description="Run GeoIntel golden QA/QC benchmark.")
     parser.add_argument("--json", action="store_true", help="Print machine-readable JSON only.")