diff --git a/CHANGELOG.md b/CHANGELOG.md index d3e50565..1d940725 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,14 @@ # Changelog +## Sprint 111 QA feature evidence persistence (2026-06-25) + +- Added feature-level QA evidence to dataset, detection and segmentation QA matching. +- Persisted matched feature ids, false-positive feature ids and false-negative feature ids inside `quality_checks.findings_json`. +- Extended QA/QC drilldown with compact matched/false-positive/false-negative feature id lists beside the existing metrics and raw findings JSON. +- Updated API contracts to document `match_evidence`, `false_positive_evidence` and `false_negative_evidence`. +- No migration, new table, provider fetching, AI behavior or new product domain was introduced. + ## Sprint 110 Map QA evidence drilldown (2026-06-25) - Extended the Map workspace QA/QC shortcut with inline evidence after comparing a saved derived selection dataset. diff --git a/backend/app/api/routes/qa.py b/backend/app/api/routes/qa.py index 47d62b64..2e535548 100644 --- a/backend/app/api/routes/qa.py +++ b/backend/app/api/routes/qa.py @@ -59,6 +59,9 @@ def compare_candidate_with_reference( "warnings": result_json.get("warnings", []), "unsupported_geometry": result_json.get("unsupported_geometry", False), "unsupported_geometries": result_json.get("unsupported_geometries", []), + "match_evidence": result_json.get("match_evidence", []), + "false_positive_evidence": result_json.get("false_positive_evidence", []), + "false_negative_evidence": result_json.get("false_negative_evidence", []), }, metrics={ "precision": result_json.get("precision"), diff --git a/backend/app/schemas/qa.py b/backend/app/schemas/qa.py index 17ce192e..076afc11 100644 --- a/backend/app/schemas/qa.py +++ b/backend/app/schemas/qa.py @@ -28,6 +28,9 @@ class QaProviderComparisonResult(BaseModel): iou_threshold: float unsupported_geometry: bool = False unsupported_geometries: list[str] = Field(default_factory=list) + match_evidence: list[dict] = Field(default_factory=list) + false_positive_evidence: list[dict] = Field(default_factory=list) + false_negative_evidence: list[dict] = Field(default_factory=list) generated_at: datetime diff --git a/backend/app/services/detection_service.py b/backend/app/services/detection_service.py index 1d9c06b0..0cde32fe 100644 --- a/backend/app/services/detection_service.py +++ b/backend/app/services/detection_service.py @@ -287,18 +287,18 @@ class DetectionService: candidate_geometries = [({"id": str(row.id), "class_name": row.class_name}, to_shape(row.geometry)) for row in detections] reference_geometries = [({"id": str(row.id), "feature_class": row.feature_class}, to_shape(row.geometry)) for row in references] - matches, false_positives, false_negatives, match_iou_values, warnings, unsupported = QaService._match_io_u_metrics( + evidence = QaService._match_io_u_evidence( candidate_geometries, reference_geometries, iou_threshold, ) - mean_iou = None if not match_iou_values else sum(match_iou_values) / len(match_iou_values) - precision = matches / (matches + false_positives) if matches + false_positives > 0 else None - recall = matches / (matches + false_negatives) if matches + false_negatives > 0 else None + mean_iou = None if not evidence.match_iou_values else sum(evidence.match_iou_values) / len(evidence.match_iou_values) + precision = evidence.matches / (evidence.matches + evidence.false_positives) if evidence.matches + evidence.false_positives > 0 else None + recall = evidence.matches / (evidence.matches + evidence.false_negatives) if evidence.matches + evidence.false_negatives > 0 else None f1_score = None if precision is not None and recall is not None: f1_score = (2 * precision * recall) / (precision + recall) if precision + recall > 0 else 0.0 - status = "unsupported" if unsupported else "ok" + status = "unsupported" if evidence.unsupported else "ok" quality_check = QualityService.persist_quality_check( db=db, project_id=run.project_id, @@ -316,19 +316,22 @@ class DetectionService: "min_confidence": min_confidence, }, findings={ - "matches": matches, - "false_positives": false_positives, - "false_negatives": false_negatives, - "warnings": warnings, - "unsupported_geometry": unsupported, + "matches": evidence.matches, + "false_positives": evidence.false_positives, + "false_negatives": evidence.false_negatives, + "warnings": evidence.warnings, + "unsupported_geometry": evidence.unsupported, + "match_evidence": evidence.match_evidence, + "false_positive_evidence": evidence.false_positive_evidence, + "false_negative_evidence": evidence.false_negative_evidence, }, metrics={ "precision": precision, "recall": recall, "f1": f1_score, "mean_iou": mean_iou, - "false_positive_count": false_positives, - "false_negative_count": false_negatives, + "false_positive_count": evidence.false_positives, + "false_negative_count": evidence.false_negatives, }, ) return { @@ -338,15 +341,18 @@ class DetectionService: "reference_dataset_id": str(reference_dataset_id), "candidate_feature_count": len(candidate_geometries), "reference_feature_count": len(reference_geometries), - "matches": matches, - "false_positives": false_positives, - "false_negatives": false_negatives, + "matches": evidence.matches, + "false_positives": evidence.false_positives, + "false_negatives": evidence.false_negatives, "precision": precision, "recall": recall, "f1_score": f1_score, "mean_iou": mean_iou, "iou_threshold": iou_threshold, - "warnings": warnings, + "warnings": evidence.warnings, + "match_evidence": evidence.match_evidence, + "false_positive_evidence": evidence.false_positive_evidence, + "false_negative_evidence": evidence.false_negative_evidence, } @staticmethod diff --git a/backend/app/services/qa_service.py b/backend/app/services/qa_service.py index d6a5112f..b67fe9d1 100644 --- a/backend/app/services/qa_service.py +++ b/backend/app/services/qa_service.py @@ -1,5 +1,6 @@ from __future__ import annotations +from dataclasses import dataclass, field from datetime import datetime, timezone from typing import Any from uuid import UUID @@ -16,6 +17,19 @@ from app.schemas.qa import QaProviderComparisonResult from app.services.vector_operations_service import VectorOperationsService +@dataclass +class QaMatchEvidence: + matches: int = 0 + false_positives: int = 0 + false_negatives: int = 0 + match_iou_values: list[float] = field(default_factory=list) + warnings: list[str] = field(default_factory=list) + unsupported: bool = False + match_evidence: list[dict[str, Any]] = field(default_factory=list) + false_positive_evidence: list[dict[str, Any]] = field(default_factory=list) + false_negative_evidence: list[dict[str, Any]] = field(default_factory=list) + + def _extract_crs_warnings(source_dataset: Dataset, reference_dataset: Dataset) -> list[str]: warnings: list[str] = [] for dataset, label in ((source_dataset, "candidate"), (reference_dataset, "reference")): @@ -92,17 +106,34 @@ class QaService: return area_geometry @staticmethod - def _match_io_u_metrics( + def _feature_identifier(feature: dict[str, Any], fallback_prefix: str, index: int) -> str: + feature_id = feature.get("id") + if feature_id is not None: + return str(feature_id) + properties = feature.get("properties") + if isinstance(properties, dict): + for key in ("vector_feature_id", "source_feature_id", "detection_id", "segmentation_id", "id", "name"): + value = properties.get(key) + if value is not None: + return str(value) + for key in ("vector_feature_id", "source_feature_id", "detection_id", "segmentation_id", "class_name", "feature_class"): + value = feature.get(key) + if value is not None: + return str(value) + return f"{fallback_prefix}-{index + 1}" + + @staticmethod + def _match_io_u_evidence( source_geometries: list[tuple[dict[str, Any], BaseGeometry]], reference_geometries: list[tuple[dict[str, Any], BaseGeometry]], iou_threshold: float, - ) -> tuple[int, int, int, list[float], list[str], bool]: + ) -> QaMatchEvidence: source_supported = [ - (feature, geom) for feature, geom in source_geometries if geom.geom_type in QaService.SUPPORTED_GEOMETRY_TYPES + (index, feature, geom) for index, (feature, geom) in enumerate(source_geometries) if geom.geom_type in QaService.SUPPORTED_GEOMETRY_TYPES ] reference_supported = [ - (feature, geom) - for feature, geom in reference_geometries + (index, feature, geom) + for index, (feature, geom) in enumerate(reference_geometries) if geom.geom_type in QaService.SUPPORTED_GEOMETRY_TYPES ] @@ -114,29 +145,35 @@ class QaService: } ) if not source_supported or not reference_supported: - return ( - 0, - len(source_supported), - len(reference_supported), - [], - [f"Unsupported geometry types: {unsupported}"] if unsupported else [], - True, + return QaMatchEvidence( + false_positives=len(source_supported), + false_negatives=len(reference_supported), + warnings=[f"Unsupported geometry types: {unsupported}"] if unsupported else [], + unsupported=True, + false_positive_evidence=[ + {"candidate_feature_id": QaService._feature_identifier(feature, "candidate", source_index)} + for source_index, feature, _ in source_supported + ], + false_negative_evidence=[ + {"reference_feature_id": QaService._feature_identifier(feature, "reference", reference_index)} + for reference_index, feature, _ in reference_supported + ], ) unmatched_reference_indices = set(range(len(reference_supported))) - matches = 0 - match_iou_values: list[float] = [] - false_positives = 0 + evidence = QaMatchEvidence(warnings=[f"Unsupported geometry types: {unsupported}"] if unsupported else [], unsupported=bool(unsupported)) - for _, source_geom in source_supported: + for source_index, source_feature, source_geom in source_supported: + source_feature_id = QaService._feature_identifier(source_feature, "candidate", source_index) if source_geom.area <= 0: - false_positives += 1 + evidence.false_positives += 1 + evidence.false_positive_evidence.append({"candidate_feature_id": source_feature_id}) continue best_iou = 0.0 best_index = None for reference_index in list(unmatched_reference_indices): - _, reference_geom = reference_supported[reference_index] + _, _, reference_geom = reference_supported[reference_index] if reference_geom.area <= 0: unmatched_reference_indices.discard(reference_index) continue @@ -161,16 +198,45 @@ class QaService: best_index = reference_index if best_index is not None and best_iou >= iou_threshold: - matches += 1 - match_iou_values.append(best_iou) + reference_original_index, reference_feature, _ = reference_supported[best_index] + evidence.matches += 1 + evidence.match_iou_values.append(best_iou) + evidence.match_evidence.append( + { + "candidate_feature_id": source_feature_id, + "reference_feature_id": QaService._feature_identifier(reference_feature, "reference", reference_original_index), + "iou": best_iou, + } + ) unmatched_reference_indices.discard(best_index) else: - false_positives += 1 + evidence.false_positives += 1 + evidence.false_positive_evidence.append({"candidate_feature_id": source_feature_id}) - false_negatives = len(unmatched_reference_indices) - warnings: list[str] = [f"Unsupported geometry types: {unsupported}"] if unsupported else [] + evidence.false_negatives = len(unmatched_reference_indices) + for reference_index in sorted(unmatched_reference_indices): + reference_original_index, reference_feature, _ = reference_supported[reference_index] + evidence.false_negative_evidence.append( + {"reference_feature_id": QaService._feature_identifier(reference_feature, "reference", reference_original_index)} + ) - return matches, false_positives, false_negatives, match_iou_values, warnings, bool(unsupported) + return evidence + + @staticmethod + def _match_io_u_metrics( + source_geometries: list[tuple[dict[str, Any], BaseGeometry]], + reference_geometries: list[tuple[dict[str, Any], BaseGeometry]], + iou_threshold: float, + ) -> tuple[int, int, int, list[float], list[str], bool]: + evidence = QaService._match_io_u_evidence(source_geometries, reference_geometries, iou_threshold) + return ( + evidence.matches, + evidence.false_positives, + evidence.false_negatives, + evidence.match_iou_values, + evidence.warnings, + evidence.unsupported, + ) @staticmethod def compare_candidate_with_reference( @@ -206,7 +272,7 @@ class QaService: candidate_geometries = QaService._apply_area_filter(candidate_geometries, area_geometry, dataset_id=candidate_dataset.id) reference_geometries = QaService._apply_area_filter(reference_geometries, area_geometry, dataset_id=reference_dataset.id) - matches, false_positives, false_negatives, match_iou_values, warnings, unsupported = QaService._match_io_u_metrics( + evidence = QaService._match_io_u_evidence( candidate_geometries, reference_geometries, iou_threshold, @@ -214,35 +280,38 @@ class QaService: candidate_feature_count = len(candidate_payload.get("features", [])) if isinstance(candidate_payload, dict) else 0 reference_feature_count = len(reference_payload.get("features", [])) if isinstance(reference_payload, dict) else 0 - mean_iou = None if not match_iou_values else sum(match_iou_values) / len(match_iou_values) + mean_iou = None if not evidence.match_iou_values else sum(evidence.match_iou_values) / len(evidence.match_iou_values) precision = None - if matches + false_positives > 0: - precision = matches / (matches + false_positives) + if evidence.matches + evidence.false_positives > 0: + precision = evidence.matches / (evidence.matches + evidence.false_positives) recall = None - if matches + false_negatives > 0: - recall = matches / (matches + false_negatives) + if evidence.matches + evidence.false_negatives > 0: + recall = evidence.matches / (evidence.matches + evidence.false_negatives) f1_score = None if precision is not None and recall is not None and precision + recall > 0: f1_score = (2 * precision * recall) / (precision + recall) - status = "unsupported" if unsupported else "ok" + status = "unsupported" if evidence.unsupported else "ok" return QaProviderComparisonResult( status=status, - warnings=_extract_crs_warnings(candidate_dataset, reference_dataset) + warnings, + warnings=_extract_crs_warnings(candidate_dataset, reference_dataset) + evidence.warnings, candidate_feature_count=candidate_feature_count, reference_feature_count=reference_feature_count, - matches=matches, - false_positives=false_positives, - false_negatives=false_negatives, + matches=evidence.matches, + false_positives=evidence.false_positives, + false_negatives=evidence.false_negatives, precision=precision, recall=recall, f1_score=f1_score, mean_iou=mean_iou, iou_threshold=iou_threshold, - unsupported_geometry=unsupported, - unsupported_geometries=warnings, + unsupported_geometry=evidence.unsupported, + unsupported_geometries=evidence.warnings, + match_evidence=evidence.match_evidence, + false_positive_evidence=evidence.false_positive_evidence, + false_negative_evidence=evidence.false_negative_evidence, generated_at=datetime.now(timezone.utc), ) diff --git a/backend/app/services/segmentation_service.py b/backend/app/services/segmentation_service.py index 93b45f11..f55d0a19 100644 --- a/backend/app/services/segmentation_service.py +++ b/backend/app/services/segmentation_service.py @@ -251,18 +251,18 @@ class SegmentationService: candidate_geometries = [({"id": str(row.id), "class_name": row.class_name}, to_shape(row.geometry)) for row in segmentations] reference_geometries = [({"id": str(row.id), "feature_class": row.feature_class}, to_shape(row.geometry)) for row in references] - matches, false_positives, false_negatives, match_iou_values, warnings, unsupported = QaService._match_io_u_metrics( + evidence = QaService._match_io_u_evidence( candidate_geometries, reference_geometries, iou_threshold, ) - mean_iou = None if not match_iou_values else sum(match_iou_values) / len(match_iou_values) - precision = matches / (matches + false_positives) if matches + false_positives > 0 else None - recall = matches / (matches + false_negatives) if matches + false_negatives > 0 else None + mean_iou = None if not evidence.match_iou_values else sum(evidence.match_iou_values) / len(evidence.match_iou_values) + precision = evidence.matches / (evidence.matches + evidence.false_positives) if evidence.matches + evidence.false_positives > 0 else None + recall = evidence.matches / (evidence.matches + evidence.false_negatives) if evidence.matches + evidence.false_negatives > 0 else None f1_score = None if precision is not None and recall is not None: f1_score = (2 * precision * recall) / (precision + recall) if precision + recall > 0 else 0.0 - status = "unsupported" if unsupported else "ok" + status = "unsupported" if evidence.unsupported else "ok" quality_check = QualityService.persist_quality_check( db=db, project_id=run.project_id, @@ -280,19 +280,22 @@ class SegmentationService: "min_confidence": min_confidence, }, findings={ - "matches": matches, - "false_positives": false_positives, - "false_negatives": false_negatives, - "warnings": warnings, - "unsupported_geometry": unsupported, + "matches": evidence.matches, + "false_positives": evidence.false_positives, + "false_negatives": evidence.false_negatives, + "warnings": evidence.warnings, + "unsupported_geometry": evidence.unsupported, + "match_evidence": evidence.match_evidence, + "false_positive_evidence": evidence.false_positive_evidence, + "false_negative_evidence": evidence.false_negative_evidence, }, metrics={ "precision": precision, "recall": recall, "f1": f1_score, "mean_iou": mean_iou, - "false_positive_count": false_positives, - "false_negative_count": false_negatives, + "false_positive_count": evidence.false_positives, + "false_negative_count": evidence.false_negatives, }, ) return { @@ -302,15 +305,18 @@ class SegmentationService: "reference_dataset_id": str(reference_dataset_id), "candidate_feature_count": len(candidate_geometries), "reference_feature_count": len(reference_geometries), - "matches": matches, - "false_positives": false_positives, - "false_negatives": false_negatives, + "matches": evidence.matches, + "false_positives": evidence.false_positives, + "false_negatives": evidence.false_negatives, "precision": precision, "recall": recall, "f1_score": f1_score, "mean_iou": mean_iou, "iou_threshold": iou_threshold, - "warnings": warnings, + "warnings": evidence.warnings, + "match_evidence": evidence.match_evidence, + "false_positive_evidence": evidence.false_positive_evidence, + "false_negative_evidence": evidence.false_negative_evidence, } @staticmethod diff --git a/backend/tests/test_qa_service.py b/backend/tests/test_qa_service.py index e2002dc2..aff0ae0e 100644 --- a/backend/tests/test_qa_service.py +++ b/backend/tests/test_qa_service.py @@ -22,23 +22,30 @@ class FakeSession: return None +def _feature(feature_id: str, coordinates: list[list[list[float]]]) -> dict: + return { + "type": "Feature", + "id": feature_id, + "properties": {"source_feature_id": feature_id}, + "geometry": { + "type": "Polygon", + "coordinates": coordinates, + }, + } + + def _write_dataset(path: Path, coordinates: list[list[list[float]]]) -> None: payload = { "type": "FeatureCollection", - "features": [ - { - "type": "Feature", - "properties": {}, - "geometry": { - "type": "Polygon", - "coordinates": coordinates, - }, - }, - ], + "features": [_feature("feature-1", coordinates)], } path.write_text(json.dumps(payload), encoding="utf-8") +def _write_features(path: Path, features: list[dict]) -> None: + path.write_text(json.dumps({"type": "FeatureCollection", "features": features}), encoding="utf-8") + + def test_qa_compare_candidate_with_reference_returns_metrics(tmp_path) -> None: project_id = uuid4() candidate_id = uuid4() @@ -87,6 +94,62 @@ def test_qa_compare_candidate_with_reference_returns_metrics(tmp_path) -> None: assert result.f1_score == 1.0 +def test_qa_compare_candidate_with_reference_returns_feature_level_evidence(tmp_path) -> None: + project_id = uuid4() + candidate_id = uuid4() + reference_id = uuid4() + candidate_path = tmp_path / "candidate.geojson" + reference_path = tmp_path / "reference.geojson" + matched_candidate = [[[4.0, 51.0], [4.1, 51.0], [4.1, 51.1], [4.0, 51.1], [4.0, 51.0]]] + matched_reference = [[[4.0, 51.0], [4.1, 51.0], [4.1, 51.1], [4.0, 51.1], [4.0, 51.0]]] + false_positive = [[[4.5, 51.5], [4.6, 51.5], [4.6, 51.6], [4.5, 51.6], [4.5, 51.5]]] + false_negative = [[[4.8, 51.8], [4.9, 51.8], [4.9, 51.9], [4.8, 51.9], [4.8, 51.8]]] + _write_features(candidate_path, [_feature("candidate-match", matched_candidate), _feature("candidate-extra", false_positive)]) + _write_features(reference_path, [_feature("reference-match", matched_reference), _feature("reference-missing", false_negative)]) + + candidate = Dataset( + id=candidate_id, + project_id=project_id, + name="candidate.geojson", + dataset_type="vector", + source="test", + storage_path=str(candidate_path), + crs="EPSG:4326", + metadata_json={"crs_assumed": False}, + ) + reference = Dataset( + id=reference_id, + project_id=project_id, + name="reference.geojson", + dataset_type="vector", + source="test", + storage_path=str(reference_path), + crs="EPSG:4326", + metadata_json={"crs_assumed": False}, + ) + + result = QaService.compare_candidate_with_reference( + db=FakeSession([candidate, reference]), + project_id=project_id, + candidate_dataset_id=candidate_id, + reference_dataset_id=reference_id, + iou_threshold=0.5, + ) + + assert result.matches == 1 + assert result.false_positives == 1 + assert result.false_negatives == 1 + assert result.match_evidence == [ + { + "candidate_feature_id": "candidate-match", + "reference_feature_id": "reference-match", + "iou": 1.0, + } + ] + assert result.false_positive_evidence == [{"candidate_feature_id": "candidate-extra"}] + assert result.false_negative_evidence == [{"reference_feature_id": "reference-missing"}] + + def test_dataset_reference_metadata_migration_declares_required_columns() -> None: migration_path = Path(__file__).parents[1] / "alembic" / "versions" / "202606120001_add_dataset_reference_metadata.py" migration_text = migration_path.read_text(encoding="utf-8") diff --git a/backend/tests/test_sprint111_qa_feature_evidence.py b/backend/tests/test_sprint111_qa_feature_evidence.py new file mode 100644 index 00000000..72aedf55 --- /dev/null +++ b/backend/tests/test_sprint111_qa_feature_evidence.py @@ -0,0 +1,39 @@ +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[2] + + +def read_text(relative_path: str) -> str: + return (ROOT / relative_path).read_text(encoding="utf-8") + + +def test_qa_feature_evidence_contract_is_documented_and_rendered() -> None: + schema = read_text("backend/app/schemas/qa.py") + quality_panel = read_text("frontend/src/components/quality/QualityResultsPanel.tsx") + api_contracts = read_text("docs/API_CONTRACTS.md") + + for field_name in ("match_evidence", "false_positive_evidence", "false_negative_evidence"): + assert field_name in schema + assert field_name in quality_panel + assert field_name in api_contracts + + assert "Feature-level QA/QC evidence" in quality_panel + assert "Matched feature ids" in quality_panel + assert "False positive feature ids" in quality_panel + assert "False negative feature ids" in quality_panel + assert "evidenceLabel" in quality_panel + + +def test_qa_services_persist_feature_evidence_without_new_migrations() -> None: + qa_route = read_text("backend/app/api/routes/qa.py") + detection_service = read_text("backend/app/services/detection_service.py") + segmentation_service = read_text("backend/app/services/segmentation_service.py") + migrations = "\n".join(path.name for path in (ROOT / "backend" / "alembic" / "versions").glob("*.py")) + + for field_name in ("match_evidence", "false_positive_evidence", "false_negative_evidence"): + assert field_name in qa_route + assert field_name in detection_service + assert field_name in segmentation_service + + assert "quality_check_items" not in migrations diff --git a/backend/tests/test_sprint7a_persistence_foundation.py b/backend/tests/test_sprint7a_persistence_foundation.py index c1a346fb..59356eb4 100644 --- a/backend/tests/test_sprint7a_persistence_foundation.py +++ b/backend/tests/test_sprint7a_persistence_foundation.py @@ -267,6 +267,15 @@ def test_qa_route_persists_quality_check_domain_record(monkeypatch) -> None: "mean_iou": 1.0, "iou_threshold": 0.5, "warnings": [], + "match_evidence": [ + { + "candidate_feature_id": "candidate-1", + "reference_feature_id": "reference-1", + "iou": 1.0, + } + ], + "false_positive_evidence": [{"candidate_feature_id": "candidate-extra"}], + "false_negative_evidence": [{"reference_feature_id": "reference-missing"}], } }, )(), @@ -287,6 +296,9 @@ def test_qa_route_persists_quality_check_domain_record(monkeypatch) -> None: assert persisted_quality_checks[0].job_id == job_id assert persisted_quality_checks[0].candidate_dataset_id == candidate_dataset_id assert persisted_quality_checks[0].reference_dataset_id == reference_dataset_id + assert persisted_quality_checks[0].findings_json["match_evidence"][0]["candidate_feature_id"] == "candidate-1" + assert persisted_quality_checks[0].findings_json["false_positive_evidence"][0]["candidate_feature_id"] == "candidate-extra" + assert persisted_quality_checks[0].findings_json["false_negative_evidence"][0]["reference_feature_id"] == "reference-missing" assert [metric.metric_key for metric in persisted_metrics] == [ "precision", "recall", diff --git a/docs/API_CONTRACTS.md b/docs/API_CONTRACTS.md index be7ff465..5a24c349 100644 --- a/docs/API_CONTRACTS.md +++ b/docs/API_CONTRACTS.md @@ -988,6 +988,14 @@ Request: Response is wrapped in the job envelope. On success, `result_json` includes precision, recall, F1, mean IoU, false positives, false negatives and `quality_check_id`. +Sprint 111 also includes feature-level evidence arrays for map/review handoff: + +- `match_evidence`: matched candidate/reference feature ids with IoU. +- `false_positive_evidence`: unmatched candidate feature ids. +- `false_negative_evidence`: unmatched reference feature ids. + +These arrays are derived from the same persisted/source geometries used for IoU matching. They are not separate QA records yet; they are persisted inside `quality_checks.findings_json`. + Sprint 7A persists the QA/QC result as: - `jobs`: execution state. @@ -1016,7 +1024,28 @@ Response: "status": "ok", "score": 0.5, "parameters_json": {}, - "findings_json": {}, + "findings_json": { + "matches": 1, + "false_positives": 1, + "false_negatives": 1, + "match_evidence": [ + { + "candidate_feature_id": "candidate-feature-id", + "reference_feature_id": "reference-feature-id", + "iou": 0.83 + } + ], + "false_positive_evidence": [ + { + "candidate_feature_id": "candidate-extra-id" + } + ], + "false_negative_evidence": [ + { + "reference_feature_id": "reference-missing-id" + } + ] + }, "metrics": [ { "metric_key": "precision", diff --git a/docs/CODEX_EXECUTION_LOG.md b/docs/CODEX_EXECUTION_LOG.md index 11baca63..52b39f96 100644 --- a/docs/CODEX_EXECUTION_LOG.md +++ b/docs/CODEX_EXECUTION_LOG.md @@ -1,3 +1,37 @@ +## Sprint 111 QA feature evidence persistence (2026-06-25) + +Changed: +- Added feature-level evidence extraction to the shared QA IoU matcher. +- Dataset QA now returns and persists `match_evidence`, `false_positive_evidence` and `false_negative_evidence`. +- Detection QA and segmentation QA now use the same evidence-aware matcher and persist the same evidence keys in `quality_checks.findings_json`. +- Extended the QA/QC drilldown with compact matched, false-positive and false-negative feature id lists before the raw findings JSON. +- Updated `docs/API_CONTRACTS.md`, `frontend/README.md`, `CHANGELOG.md` and `docs/TODO.md`. +- Added/extended regression coverage in `backend/tests/test_qa_service.py`, `backend/tests/test_sprint7a_persistence_foundation.py` and `backend/tests/test_sprint111_qa_feature_evidence.py`. + +Validation: +- RED: `python -m pytest backend\tests\test_qa_service.py -q` failed before implementation because `QaProviderComparisonResult` had no `match_evidence`. +- RED: `python -m pytest backend\tests\test_sprint111_qa_feature_evidence.py -q` failed before docs were updated because `docs/API_CONTRACTS.md` did not document the evidence keys. +- `python -m pytest backend\tests\test_qa_service.py -q` passed: 3 tests. +- `python -m pytest backend\tests\test_sprint8c_detection_visualization_qa.py backend\tests\test_sprint9_segmentation_foundation.py -q` passed: 18 tests. +- `python -m pytest backend\tests\test_sprint111_qa_feature_evidence.py -q` passed: 2 tests. +- `python -m pytest backend\tests\test_qa_service.py backend\tests\test_sprint7a_persistence_foundation.py -q` passed: 10 tests. +- `python -m compileall backend/app` passed. +- `cd backend && python -m pytest -q` passed: 352 tests. +- `cd frontend && npm run typecheck` passed. +- `cd frontend && npm run build` passed. +- `cd backend && python -m alembic heads` passed: `202606120900 (head)`. +- `cd backend && python -m alembic upgrade head --sql` passed. +- `bash -n scripts/live_migration_smoke.sh` passed. +- `bash scripts/run_readiness_check.sh` passed. + +Limitations: +- Feature-level evidence is persisted as ids/IoU metadata in `quality_checks.findings_json`; no `quality_check_items` table or first-class evidence geometry table was introduced. +- Evidence map overlays can now be built from persisted ids, but overlay generation remains future work. +- No migration, provider fetching, AI dependency, real model behavior or new product domain was added. + +Next recommended pass: +- Add QA evidence overlay generation by resolving persisted evidence ids back to candidate/reference geometries and rendering false positives/false negatives as MapLibre layers. + ## Sprint 110 Map QA evidence drilldown (2026-06-25) Changed: diff --git a/docs/TODO.md b/docs/TODO.md index 9a50f9c4..5cb7960f 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -377,3 +377,4 @@ This file now starts with the current implementation status. Older preparation/b - [x] Persist map area selections as reusable derived vector datasets indexed into `vector_features`. - [x] Add Map workspace QA/QC shortcut for saved derived selection datasets. - [x] Add Map workspace QA/QC evidence drilldown handoff for saved selection comparisons. +- [x] Persist QA/QC feature-level evidence for matches, false positives and false negatives. diff --git a/frontend/README.md b/frontend/README.md index 7bc345d8..566f3f70 100644 --- a/frontend/README.md +++ b/frontend/README.md @@ -287,6 +287,7 @@ AI Lab run controls explicitly explain when no raster dataset is available, inst - Raster controls show the latest generated tile manifest path from persisted `raster.tile` jobs and can hand that path directly to Detection Lab or Segmentation Lab with the selected raster dataset. - The QA/QC workspace shows candidate/reference handoff cards and resolves persisted quality-check dataset IDs back to dataset names when the datasets are loaded in the current project context. - The QA/QC workspace includes a selected-check evidence drilldown with candidate/reference provenance, false-positive/negative metric evidence, map handoff context and parameters/findings JSON. +- QA/QC findings now persist feature-level evidence in `findings_json`: matched candidate/reference feature ids with IoU, false-positive candidate feature ids and false-negative reference feature ids. The QA/QC drilldown renders these as compact evidence lists before the raw JSON. ## Raster dependency visibility diff --git a/frontend/src/components/quality/QualityResultsPanel.tsx b/frontend/src/components/quality/QualityResultsPanel.tsx index daef0595..e3af9839 100644 --- a/frontend/src/components/quality/QualityResultsPanel.tsx +++ b/frontend/src/components/quality/QualityResultsPanel.tsx @@ -46,6 +46,20 @@ function metricByKey(check: QualityCheckRead | null, metricKey: string): MetricR return check?.metrics.find((metric) => metric.metric_key === metricKey) } +function findingEvidenceList(check: QualityCheckRead | null, key: string): Record[] { + const value = check?.findings_json?.[key] + return Array.isArray(value) ? value.filter((item): item is Record => Boolean(item) && typeof item === 'object' && !Array.isArray(item)) : [] +} + +function evidenceLabel(item: Record): string { + const candidate = item['candidate_feature_id'] + const reference = item['reference_feature_id'] + const iou = item['iou'] + const pairLabel = [candidate ? `Candidate ${candidate}` : null, reference ? `Reference ${reference}` : null].filter(Boolean).join(' / ') + const iouValue = Number(iou) + return iou === null || iou === undefined || !Number.isFinite(iouValue) ? pairLabel || JSON.stringify(item) : `${pairLabel || 'Match'} / IoU ${iouValue.toFixed(3)}` +} + function formatQualityTimestamp(value?: string | null): string { return value || 'n/a' } @@ -108,6 +122,9 @@ export function QualityResultsPanel({ const selectedReferenceName = selectedQualityCheck ? datasetNameById.get(selectedQualityCheck.reference_dataset_id) ?? selectedQualityCheck.reference_dataset_id : 'n/a' + const selectedMatchEvidence = findingEvidenceList(selectedQualityCheck, 'match_evidence') + const selectedFalsePositiveEvidence = findingEvidenceList(selectedQualityCheck, 'false_positive_evidence') + const selectedFalseNegativeEvidence = findingEvidenceList(selectedQualityCheck, 'false_negative_evidence') const qualityStatuses = useMemo(() => Array.from(new Set(qualityChecks.map((check) => check.status))).sort(), [qualityChecks]) const qualityTypes = useMemo(() => Array.from(new Set(qualityChecks.map((check) => check.check_type))).sort(), [qualityChecks]) const filteredQualityChecks = useMemo( @@ -239,6 +256,44 @@ export function QualityResultsPanel({

Use the candidate and reference datasets as map layers for spatial review.

+
+
+ Matched feature ids + {selectedMatchEvidence.length > 0 ? ( +
    + {selectedMatchEvidence.slice(0, 6).map((item, index) => ( +
  • {evidenceLabel(item)}
    • + ))} +
+ ) : ( +

No matched feature ids persisted for this check.

+ )} +
+
+ False positive feature ids + {selectedFalsePositiveEvidence.length > 0 ? ( +
    + {selectedFalsePositiveEvidence.slice(0, 6).map((item, index) => ( +
  • {evidenceLabel(item)}
    • + ))} +
+ ) : ( +

No false-positive feature ids persisted for this check.

+ )} +
+
+ False negative feature ids + {selectedFalseNegativeEvidence.length > 0 ? ( +
    + {selectedFalseNegativeEvidence.slice(0, 6).map((item, index) => ( +
  • {evidenceLabel(item)}
    • + ))} +
+ ) : ( +

No false-negative feature ids persisted for this check.

+ )} +
+
Parameters diff --git a/frontend/src/styles/app.css b/frontend/src/styles/app.css index 057d0189..672c9fb7 100644 --- a/frontend/src/styles/app.css +++ b/frontend/src/styles/app.css @@ -1618,6 +1618,7 @@ button.entity-card { .quality-drilldown-grid, .quality-evidence-token-grid, +.quality-feature-evidence-grid, .quality-provenance-grid { display: grid; grid-template-columns: repeat(auto-fit, minmax(12rem, 1fr)); @@ -1641,6 +1642,7 @@ button.entity-card { .quality-drilldown-grid > div, .quality-evidence-token-grid > div, +.quality-feature-evidence-grid > div, .quality-provenance-grid > div { min-width: 0; border: 1px solid var(--line); @@ -1663,6 +1665,7 @@ button.entity-card { .quality-handoff-grid span, .quality-drilldown-grid span, .quality-evidence-token-grid span, +.quality-feature-evidence-grid span, .quality-provenance-grid span, .quality-score-row span, .latest-export-card span, @@ -1703,7 +1706,8 @@ button.entity-card { } .quality-drilldown-grid p, -.quality-evidence-token-grid p { +.quality-evidence-token-grid p, +.quality-feature-evidence-grid p { margin: 0.24rem 0 0; color: var(--muted); font-size: 0.8rem; @@ -1711,6 +1715,17 @@ button.entity-card { overflow-wrap: anywhere; } +.quality-feature-evidence-grid ul { + display: grid; + gap: 0.28rem; + margin: 0.38rem 0 0; + padding-left: 1.1rem; + color: var(--text); + font-size: 0.78rem; + line-height: 1.35; + overflow-wrap: anywhere; +} + .quality-provenance-pre { max-height: 11rem; margin: 0.38rem 0 0;