Jens/geointel
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Persist QA feature evidence
GeoIntel CI / docs-smoke (push) Has been cancelled
Details
GeoIntel CI / contract-smoke (push) Has been cancelled
Details

Browse Source
This commit is contained in:
Codex
2026-06-25 02:57:50 +02:00
parent 49e58cd1c7
commit 368c1ad738
15 changed files with 425 additions and 81 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/app/api/routes/qa.py
+3
View File
@@ -59,6 +59,9 @@ def compare_candidate_with_reference(
"warnings": result_json.get("warnings", []),
"unsupported_geometry": result_json.get("unsupported_geometry", False),
"unsupported_geometries": result_json.get("unsupported_geometries", []),
"match_evidence": result_json.get("match_evidence", []),
"false_positive_evidence": result_json.get("false_positive_evidence", []),
"false_negative_evidence": result_json.get("false_negative_evidence", []),
},
metrics={
"precision": result_json.get("precision"),
backend/app/schemas/qa.py
+3
View File
@@ -28,6 +28,9 @@ class QaProviderComparisonResult(BaseModel):
iou_threshold: float
unsupported_geometry: bool = False
unsupported_geometries: list[str] = Field(default_factory=list)
match_evidence: list[dict] = Field(default_factory=list)
false_positive_evidence: list[dict] = Field(default_factory=list)
false_negative_evidence: list[dict] = Field(default_factory=list)
generated_at: datetime
backend/app/services/detection_service.py
+22 -16
View File
@@ -287,18 +287,18 @@ class DetectionService:
candidate_geometries = [({"id": str(row.id), "class_name": row.class_name}, to_shape(row.geometry)) for row in detections]
reference_geometries = [({"id": str(row.id), "feature_class": row.feature_class}, to_shape(row.geometry)) for row in references]
matches, false_positives, false_negatives, match_iou_values, warnings, unsupported = QaService._match_io_u_metrics(
evidence = QaService._match_io_u_evidence(
candidate_geometries,
reference_geometries,
iou_threshold,
)
mean_iou = None if not match_iou_values else sum(match_iou_values) / len(match_iou_values)
precision = matches / (matches + false_positives) if matches + false_positives > 0 else None
recall = matches / (matches + false_negatives) if matches + false_negatives > 0 else None
mean_iou = None if not evidence.match_iou_values else sum(evidence.match_iou_values) / len(evidence.match_iou_values)
precision = evidence.matches / (evidence.matches + evidence.false_positives) if evidence.matches + evidence.false_positives > 0 else None
recall = evidence.matches / (evidence.matches + evidence.false_negatives) if evidence.matches + evidence.false_negatives > 0 else None
f1_score = None
if precision is not None and recall is not None:
f1_score = (2 * precision * recall) / (precision + recall) if precision + recall > 0 else 0.0
status = "unsupported" if unsupported else "ok"
status = "unsupported" if evidence.unsupported else "ok"
quality_check = QualityService.persist_quality_check(
db=db,
project_id=run.project_id,
@@ -316,19 +316,22 @@ class DetectionService:
"min_confidence": min_confidence,
},
findings={
"matches": matches,
"false_positives": false_positives,
"false_negatives": false_negatives,
"warnings": warnings,
"unsupported_geometry": unsupported,
"matches": evidence.matches,
"false_positives": evidence.false_positives,
"false_negatives": evidence.false_negatives,
"warnings": evidence.warnings,
"unsupported_geometry": evidence.unsupported,
"match_evidence": evidence.match_evidence,
"false_positive_evidence": evidence.false_positive_evidence,
"false_negative_evidence": evidence.false_negative_evidence,
},
metrics={
"precision": precision,
"recall": recall,
"f1": f1_score,
"mean_iou": mean_iou,
"false_positive_count": false_positives,
"false_negative_count": false_negatives,
"false_positive_count": evidence.false_positives,
"false_negative_count": evidence.false_negatives,
},
)
return {
@@ -338,15 +341,18 @@ class DetectionService:
"reference_dataset_id": str(reference_dataset_id),
"candidate_feature_count": len(candidate_geometries),
"reference_feature_count": len(reference_geometries),
"matches": matches,
"false_positives": false_positives,
"false_negatives": false_negatives,
"matches": evidence.matches,
"false_positives": evidence.false_positives,
"false_negatives": evidence.false_negatives,
"precision": precision,
"recall": recall,
"f1_score": f1_score,
"mean_iou": mean_iou,
"iou_threshold": iou_threshold,
"warnings": warnings,
"warnings": evidence.warnings,
"match_evidence": evidence.match_evidence,
"false_positive_evidence": evidence.false_positive_evidence,
"false_negative_evidence": evidence.false_negative_evidence,
}
@staticmethod
backend/app/services/qa_service.py
+106 -37
View File
@@ -1,5 +1,6 @@
from __future__ import annotations
from dataclasses import dataclass, field
from datetime import datetime, timezone
from typing import Any
from uuid import UUID
@@ -16,6 +17,19 @@ from app.schemas.qa import QaProviderComparisonResult
from app.services.vector_operations_service import VectorOperationsService
@dataclass
class QaMatchEvidence:
matches: int = 0
false_positives: int = 0
false_negatives: int = 0
match_iou_values: list[float] = field(default_factory=list)
warnings: list[str] = field(default_factory=list)
unsupported: bool = False
match_evidence: list[dict[str, Any]] = field(default_factory=list)
false_positive_evidence: list[dict[str, Any]] = field(default_factory=list)
false_negative_evidence: list[dict[str, Any]] = field(default_factory=list)
def _extract_crs_warnings(source_dataset: Dataset, reference_dataset: Dataset) -> list[str]:
warnings: list[str] = []
for dataset, label in ((source_dataset, "candidate"), (reference_dataset, "reference")):
@@ -92,17 +106,34 @@ class QaService:
return area_geometry
@staticmethod
def _match_io_u_metrics(
def _feature_identifier(feature: dict[str, Any], fallback_prefix: str, index: int) -> str:
feature_id = feature.get("id")
if feature_id is not None:
return str(feature_id)
properties = feature.get("properties")
if isinstance(properties, dict):
for key in ("vector_feature_id", "source_feature_id", "detection_id", "segmentation_id", "id", "name"):
value = properties.get(key)
if value is not None:
return str(value)
for key in ("vector_feature_id", "source_feature_id", "detection_id", "segmentation_id", "class_name", "feature_class"):
value = feature.get(key)
if value is not None:
return str(value)
return f"{fallback_prefix}-{index + 1}"
@staticmethod
def _match_io_u_evidence(
source_geometries: list[tuple[dict[str, Any], BaseGeometry]],
reference_geometries: list[tuple[dict[str, Any], BaseGeometry]],
iou_threshold: float,
) -> tuple[int, int, int, list[float], list[str], bool]:
) -> QaMatchEvidence:
source_supported = [
(feature, geom) for feature, geom in source_geometries if geom.geom_type in QaService.SUPPORTED_GEOMETRY_TYPES
(index, feature, geom) for index, (feature, geom) in enumerate(source_geometries) if geom.geom_type in QaService.SUPPORTED_GEOMETRY_TYPES
]
reference_supported = [
(feature, geom)
for feature, geom in reference_geometries
(index, feature, geom)
for index, (feature, geom) in enumerate(reference_geometries)
if geom.geom_type in QaService.SUPPORTED_GEOMETRY_TYPES
]
@@ -114,29 +145,35 @@ class QaService:
}
)
if not source_supported or not reference_supported:
return (
0,
len(source_supported),
len(reference_supported),
[],
[f"Unsupported geometry types: {unsupported}"] if unsupported else [],
True,
return QaMatchEvidence(
false_positives=len(source_supported),
false_negatives=len(reference_supported),
warnings=[f"Unsupported geometry types: {unsupported}"] if unsupported else [],
unsupported=True,
false_positive_evidence=[
{"candidate_feature_id": QaService._feature_identifier(feature, "candidate", source_index)}
for source_index, feature, _ in source_supported
],
false_negative_evidence=[
{"reference_feature_id": QaService._feature_identifier(feature, "reference", reference_index)}
for reference_index, feature, _ in reference_supported
],
)
unmatched_reference_indices = set(range(len(reference_supported)))
matches = 0
match_iou_values: list[float] = []
false_positives = 0
evidence = QaMatchEvidence(warnings=[f"Unsupported geometry types: {unsupported}"] if unsupported else [], unsupported=bool(unsupported))
for _, source_geom in source_supported:
for source_index, source_feature, source_geom in source_supported:
source_feature_id = QaService._feature_identifier(source_feature, "candidate", source_index)
if source_geom.area <= 0:
false_positives += 1
evidence.false_positives += 1
evidence.false_positive_evidence.append({"candidate_feature_id": source_feature_id})
continue
best_iou = 0.0
best_index = None
for reference_index in list(unmatched_reference_indices):
_, reference_geom = reference_supported[reference_index]
_, _, reference_geom = reference_supported[reference_index]
if reference_geom.area <= 0:
unmatched_reference_indices.discard(reference_index)
continue
@@ -161,16 +198,45 @@ class QaService:
best_index = reference_index
if best_index is not None and best_iou >= iou_threshold:
matches += 1
match_iou_values.append(best_iou)
reference_original_index, reference_feature, _ = reference_supported[best_index]
evidence.matches += 1
evidence.match_iou_values.append(best_iou)
evidence.match_evidence.append(
{
"candidate_feature_id": source_feature_id,
"reference_feature_id": QaService._feature_identifier(reference_feature, "reference", reference_original_index),
"iou": best_iou,
}
)
unmatched_reference_indices.discard(best_index)
else:
false_positives += 1
evidence.false_positives += 1
evidence.false_positive_evidence.append({"candidate_feature_id": source_feature_id})
false_negatives = len(unmatched_reference_indices)
warnings: list[str] = [f"Unsupported geometry types: {unsupported}"] if unsupported else []
evidence.false_negatives = len(unmatched_reference_indices)
for reference_index in sorted(unmatched_reference_indices):
reference_original_index, reference_feature, _ = reference_supported[reference_index]
evidence.false_negative_evidence.append(
{"reference_feature_id": QaService._feature_identifier(reference_feature, "reference", reference_original_index)}
)
return matches, false_positives, false_negatives, match_iou_values, warnings, bool(unsupported)
return evidence
@staticmethod
def _match_io_u_metrics(
source_geometries: list[tuple[dict[str, Any], BaseGeometry]],
reference_geometries: list[tuple[dict[str, Any], BaseGeometry]],
iou_threshold: float,
) -> tuple[int, int, int, list[float], list[str], bool]:
evidence = QaService._match_io_u_evidence(source_geometries, reference_geometries, iou_threshold)
return (
evidence.matches,
evidence.false_positives,
evidence.false_negatives,
evidence.match_iou_values,
evidence.warnings,
evidence.unsupported,
)
@staticmethod
def compare_candidate_with_reference(
@@ -206,7 +272,7 @@ class QaService:
candidate_geometries = QaService._apply_area_filter(candidate_geometries, area_geometry, dataset_id=candidate_dataset.id)
reference_geometries = QaService._apply_area_filter(reference_geometries, area_geometry, dataset_id=reference_dataset.id)
matches, false_positives, false_negatives, match_iou_values, warnings, unsupported = QaService._match_io_u_metrics(
evidence = QaService._match_io_u_evidence(
candidate_geometries,
reference_geometries,
iou_threshold,
@@ -214,35 +280,38 @@ class QaService:
candidate_feature_count = len(candidate_payload.get("features", [])) if isinstance(candidate_payload, dict) else 0
reference_feature_count = len(reference_payload.get("features", [])) if isinstance(reference_payload, dict) else 0
mean_iou = None if not match_iou_values else sum(match_iou_values) / len(match_iou_values)
mean_iou = None if not evidence.match_iou_values else sum(evidence.match_iou_values) / len(evidence.match_iou_values)
precision = None
if matches + false_positives > 0:
precision = matches / (matches + false_positives)
if evidence.matches + evidence.false_positives > 0:
precision = evidence.matches / (evidence.matches + evidence.false_positives)
recall = None
if matches + false_negatives > 0:
recall = matches / (matches + false_negatives)
if evidence.matches + evidence.false_negatives > 0:
recall = evidence.matches / (evidence.matches + evidence.false_negatives)
f1_score = None
if precision is not None and recall is not None and precision + recall > 0:
f1_score = (2 * precision * recall) / (precision + recall)
status = "unsupported" if unsupported else "ok"
status = "unsupported" if evidence.unsupported else "ok"
return QaProviderComparisonResult(
status=status,
warnings=_extract_crs_warnings(candidate_dataset, reference_dataset) + warnings,
warnings=_extract_crs_warnings(candidate_dataset, reference_dataset) + evidence.warnings,
candidate_feature_count=candidate_feature_count,
reference_feature_count=reference_feature_count,
matches=matches,
false_positives=false_positives,
false_negatives=false_negatives,
matches=evidence.matches,
false_positives=evidence.false_positives,
false_negatives=evidence.false_negatives,
precision=precision,
recall=recall,
f1_score=f1_score,
mean_iou=mean_iou,
iou_threshold=iou_threshold,
unsupported_geometry=unsupported,
unsupported_geometries=warnings,
unsupported_geometry=evidence.unsupported,
unsupported_geometries=evidence.warnings,
match_evidence=evidence.match_evidence,
false_positive_evidence=evidence.false_positive_evidence,
false_negative_evidence=evidence.false_negative_evidence,
generated_at=datetime.now(timezone.utc),
)
backend/app/services/segmentation_service.py
+22 -16
View File
@@ -251,18 +251,18 @@ class SegmentationService:
candidate_geometries = [({"id": str(row.id), "class_name": row.class_name}, to_shape(row.geometry)) for row in segmentations]
reference_geometries = [({"id": str(row.id), "feature_class": row.feature_class}, to_shape(row.geometry)) for row in references]
matches, false_positives, false_negatives, match_iou_values, warnings, unsupported = QaService._match_io_u_metrics(
evidence = QaService._match_io_u_evidence(
candidate_geometries,
reference_geometries,
iou_threshold,
)
mean_iou = None if not match_iou_values else sum(match_iou_values) / len(match_iou_values)
precision = matches / (matches + false_positives) if matches + false_positives > 0 else None
recall = matches / (matches + false_negatives) if matches + false_negatives > 0 else None
mean_iou = None if not evidence.match_iou_values else sum(evidence.match_iou_values) / len(evidence.match_iou_values)
precision = evidence.matches / (evidence.matches + evidence.false_positives) if evidence.matches + evidence.false_positives > 0 else None
recall = evidence.matches / (evidence.matches + evidence.false_negatives) if evidence.matches + evidence.false_negatives > 0 else None
f1_score = None
if precision is not None and recall is not None:
f1_score = (2 * precision * recall) / (precision + recall) if precision + recall > 0 else 0.0
status = "unsupported" if unsupported else "ok"
status = "unsupported" if evidence.unsupported else "ok"
quality_check = QualityService.persist_quality_check(
db=db,
project_id=run.project_id,
@@ -280,19 +280,22 @@ class SegmentationService:
"min_confidence": min_confidence,
},
findings={
"matches": matches,
"false_positives": false_positives,
"false_negatives": false_negatives,
"warnings": warnings,
"unsupported_geometry": unsupported,
"matches": evidence.matches,
"false_positives": evidence.false_positives,
"false_negatives": evidence.false_negatives,
"warnings": evidence.warnings,
"unsupported_geometry": evidence.unsupported,
"match_evidence": evidence.match_evidence,
"false_positive_evidence": evidence.false_positive_evidence,
"false_negative_evidence": evidence.false_negative_evidence,
},
metrics={
"precision": precision,
"recall": recall,
"f1": f1_score,
"mean_iou": mean_iou,
"false_positive_count": false_positives,
"false_negative_count": false_negatives,
"false_positive_count": evidence.false_positives,
"false_negative_count": evidence.false_negatives,
},
)
return {
@@ -302,15 +305,18 @@ class SegmentationService:
"reference_dataset_id": str(reference_dataset_id),
"candidate_feature_count": len(candidate_geometries),
"reference_feature_count": len(reference_geometries),
"matches": matches,
"false_positives": false_positives,
"false_negatives": false_negatives,
"matches": evidence.matches,
"false_positives": evidence.false_positives,
"false_negatives": evidence.false_negatives,
"precision": precision,
"recall": recall,
"f1_score": f1_score,
"mean_iou": mean_iou,
"iou_threshold": iou_threshold,
"warnings": warnings,
"warnings": evidence.warnings,
"match_evidence": evidence.match_evidence,
"false_positive_evidence": evidence.false_positive_evidence,
"false_negative_evidence": evidence.false_negative_evidence,
}
@staticmethod
backend/tests/test_qa_service.py
+73 -10
View File
@@ -22,23 +22,30 @@ class FakeSession:
return None
def _feature(feature_id: str, coordinates: list[list[list[float]]]) -> dict:
return {
"type": "Feature",
"id": feature_id,
"properties": {"source_feature_id": feature_id},
"geometry": {
"type": "Polygon",
"coordinates": coordinates,
},
}
def _write_dataset(path: Path, coordinates: list[list[list[float]]]) -> None:
payload = {
"type": "FeatureCollection",
"features": [
{
"type": "Feature",
"properties": {},
"geometry": {
"type": "Polygon",
"coordinates": coordinates,
},
},
],
"features": [_feature("feature-1", coordinates)],
}
path.write_text(json.dumps(payload), encoding="utf-8")
def _write_features(path: Path, features: list[dict]) -> None:
path.write_text(json.dumps({"type": "FeatureCollection", "features": features}), encoding="utf-8")
def test_qa_compare_candidate_with_reference_returns_metrics(tmp_path) -> None:
project_id = uuid4()
candidate_id = uuid4()
@@ -87,6 +94,62 @@ def test_qa_compare_candidate_with_reference_returns_metrics(tmp_path) -> None:
assert result.f1_score == 1.0
def test_qa_compare_candidate_with_reference_returns_feature_level_evidence(tmp_path) -> None:
project_id = uuid4()
candidate_id = uuid4()
reference_id = uuid4()
candidate_path = tmp_path / "candidate.geojson"
reference_path = tmp_path / "reference.geojson"
matched_candidate = [[[4.0, 51.0], [4.1, 51.0], [4.1, 51.1], [4.0, 51.1], [4.0, 51.0]]]
matched_reference = [[[4.0, 51.0], [4.1, 51.0], [4.1, 51.1], [4.0, 51.1], [4.0, 51.0]]]
false_positive = [[[4.5, 51.5], [4.6, 51.5], [4.6, 51.6], [4.5, 51.6], [4.5, 51.5]]]
false_negative = [[[4.8, 51.8], [4.9, 51.8], [4.9, 51.9], [4.8, 51.9], [4.8, 51.8]]]
_write_features(candidate_path, [_feature("candidate-match", matched_candidate), _feature("candidate-extra", false_positive)])
_write_features(reference_path, [_feature("reference-match", matched_reference), _feature("reference-missing", false_negative)])
candidate = Dataset(
id=candidate_id,
project_id=project_id,
name="candidate.geojson",
dataset_type="vector",
source="test",
storage_path=str(candidate_path),
crs="EPSG:4326",
metadata_json={"crs_assumed": False},
)
reference = Dataset(
id=reference_id,
project_id=project_id,
name="reference.geojson",
dataset_type="vector",
source="test",
storage_path=str(reference_path),
crs="EPSG:4326",
metadata_json={"crs_assumed": False},
)
result = QaService.compare_candidate_with_reference(
db=FakeSession([candidate, reference]),
project_id=project_id,
candidate_dataset_id=candidate_id,
reference_dataset_id=reference_id,
iou_threshold=0.5,
)
assert result.matches == 1
assert result.false_positives == 1
assert result.false_negatives == 1
assert result.match_evidence == [
{
"candidate_feature_id": "candidate-match",
"reference_feature_id": "reference-match",
"iou": 1.0,
}
]
assert result.false_positive_evidence == [{"candidate_feature_id": "candidate-extra"}]
assert result.false_negative_evidence == [{"reference_feature_id": "reference-missing"}]
def test_dataset_reference_metadata_migration_declares_required_columns() -> None:
migration_path = Path(__file__).parents[1] / "alembic" / "versions" / "202606120001_add_dataset_reference_metadata.py"
migration_text = migration_path.read_text(encoding="utf-8")
backend/tests/test_sprint111_qa_feature_evidence.py
+39
View File
@@ -0,0 +1,39 @@
from pathlib import Path
ROOT = Path(__file__).resolve().parents[2]
def read_text(relative_path: str) -> str:
return (ROOT / relative_path).read_text(encoding="utf-8")
def test_qa_feature_evidence_contract_is_documented_and_rendered() -> None:
schema = read_text("backend/app/schemas/qa.py")
quality_panel = read_text("frontend/src/components/quality/QualityResultsPanel.tsx")
api_contracts = read_text("docs/API_CONTRACTS.md")
for field_name in ("match_evidence", "false_positive_evidence", "false_negative_evidence"):
assert field_name in schema
assert field_name in quality_panel
assert field_name in api_contracts
assert "Feature-level QA/QC evidence" in quality_panel
assert "Matched feature ids" in quality_panel
assert "False positive feature ids" in quality_panel
assert "False negative feature ids" in quality_panel
assert "evidenceLabel" in quality_panel
def test_qa_services_persist_feature_evidence_without_new_migrations() -> None:
qa_route = read_text("backend/app/api/routes/qa.py")
detection_service = read_text("backend/app/services/detection_service.py")
segmentation_service = read_text("backend/app/services/segmentation_service.py")
migrations = "\n".join(path.name for path in (ROOT / "backend" / "alembic" / "versions").glob("*.py"))
for field_name in ("match_evidence", "false_positive_evidence", "false_negative_evidence"):
assert field_name in qa_route
assert field_name in detection_service
assert field_name in segmentation_service
assert "quality_check_items" not in migrations
backend/tests/test_sprint7a_persistence_foundation.py
+12
View File
@@ -267,6 +267,15 @@ def test_qa_route_persists_quality_check_domain_record(monkeypatch) -> None:
"mean_iou": 1.0,
"iou_threshold": 0.5,
"warnings": [],
"match_evidence": [
{
"candidate_feature_id": "candidate-1",
"reference_feature_id": "reference-1",
"iou": 1.0,
}
],
"false_positive_evidence": [{"candidate_feature_id": "candidate-extra"}],
"false_negative_evidence": [{"reference_feature_id": "reference-missing"}],
}
},
)(),
@@ -287,6 +296,9 @@ def test_qa_route_persists_quality_check_domain_record(monkeypatch) -> None:
assert persisted_quality_checks[0].job_id == job_id
assert persisted_quality_checks[0].candidate_dataset_id == candidate_dataset_id
assert persisted_quality_checks[0].reference_dataset_id == reference_dataset_id
assert persisted_quality_checks[0].findings_json["match_evidence"][0]["candidate_feature_id"] == "candidate-1"
assert persisted_quality_checks[0].findings_json["false_positive_evidence"][0]["candidate_feature_id"] == "candidate-extra"
assert persisted_quality_checks[0].findings_json["false_negative_evidence"][0]["reference_feature_id"] == "reference-missing"
assert [metric.metric_key for metric in persisted_metrics] == [
"precision",
"recall",