from __future__ import annotations import json from datetime import UTC, datetime from uuid import uuid4 import pytest from fastapi.testclient import TestClient from geoalchemy2.shape import from_shape from shapely.geometry import Polygon, box from app.core.errors import AppError from app.main import app from app.db.session import get_db from app.models import AnalysisRun, Dataset, Detection, Metric, QualityCheck, SourceRegistry, SourceSnapshot, VectorFeature from app.services.detection_service import DetectionService class FakeQuery: def __init__(self, rows): self.rows = list(rows) def filter(self, *criteria): for criterion in criteria: left = getattr(criterion, "left", None) right = getattr(criterion, "right", None) operator = getattr(criterion, "operator", None) name = getattr(left, "name", None) value = getattr(right, "value", right) if name and operator: if operator.__name__ == "eq": self.rows = [row for row in self.rows if getattr(row, name) == value] elif operator.__name__ == "ge": self.rows = [row for row in self.rows if getattr(row, name) >= value] return self def order_by(self, *_args): return self def all(self): return list(self.rows) def first(self): return self.rows[0] if self.rows else None def count(self): return len(self.rows) class FakeSession: def __init__(self, objects=None, query_rows=None) -> None: self.objects = objects or {} self.query_rows = query_rows or {} self.added = [] self.commits = 0 self.refreshes = [] def get(self, model, item_id): return self.objects.get((model, item_id)) def query(self, model): return FakeQuery(self.query_rows.get(model, [])) def add(self, item) -> None: self.added.append(item) if getattr(item, "id", None) is not None: self.objects[(item.__class__, item.id)] = item def commit(self) -> None: self.commits += 1 def refresh(self, item) -> None: self.refreshes.append(item) def _detection(project_id, dataset_id, analysis_run_id, class_name="building", confidence=0.91, geom=None): return Detection( id=uuid4(), project_id=project_id, dataset_id=dataset_id, analysis_run_id=analysis_run_id, job_id=uuid4(), model_name="yolo-configured", model_version="local-test", class_name=class_name, confidence=confidence, geometry=from_shape(geom or box(4.0, 51.0, 4.1, 51.1), srid=4326), bbox_json={"x_min": 1, "y_min": 2, "x_max": 10, "y_max": 12}, source_tile_path="storage/tiles/tile_0000.tif", ) def _source_dataset(project_id, dataset_id): return Dataset( id=dataset_id, project_id=project_id, name="source.tif", dataset_type="raster", source="test", source_name="test", ) def _authoritative_reference(dataset: Dataset) -> Dataset: """Model the reference as a fully governed GRB fixture, never test data.""" source_id = uuid4() snapshot_id = uuid4() checksum = "a" * 64 source = SourceRegistry( id=source_id, source_key="grb", display_name="GRB QA fixture", classification="authoritative", authority_name="Digitaal Vlaanderen", authority_scope_json={"zone": "Flanders"}, usage_policy_json={"ground_truth_allowed": True, "validation_authority": {"building_validation": "primary"}}, ) snapshot = SourceSnapshot( id=snapshot_id, source_registry_id=source_id, snapshot_key=f"detection-qa-{dataset.id}", checksum_sha256=checksum, ingest_status="ingested", freshness_status="current", ) dataset.source = "grb" dataset.source_name = "grb" dataset.dataset_role = "reference" dataset.status = "ready" dataset.checksum_sha256 = checksum dataset.source_registry_id = source_id dataset.source_snapshot_id = snapshot_id dataset.data_contract_key = "geointel.vector.geojson" dataset.data_contract_version = "1.0.0" dataset.validation_status = "passed" dataset.provenance_status = "complete" dataset.lineage_status = "complete" dataset.quarantine_status = "not_quarantined" dataset.source_registry = source dataset.source_snapshot = snapshot return dataset def test_detection_geojson_feature_collection_shape() -> None: project_id = uuid4() dataset_id = uuid4() analysis_run_id = uuid4() detection = _detection(project_id, dataset_id, analysis_run_id) db = FakeSession( objects={(AnalysisRun, analysis_run_id): AnalysisRun(id=analysis_run_id, project_id=project_id, analysis_type="detection", status="success", parameters_json={})}, query_rows={Detection: [detection]}, ) feature_collection = DetectionService.detections_to_geojson(db, analysis_run_id=analysis_run_id) assert feature_collection["type"] == "FeatureCollection" assert len(feature_collection["features"]) == 1 feature = feature_collection["features"][0] assert feature["geometry"]["type"] == "Polygon" assert feature["properties"]["detection_id"] == str(detection.id) assert feature["properties"]["class_name"] == "building" assert feature["properties"]["confidence"] == 0.91 assert feature["properties"]["model_name"] == "yolo-configured" assert feature["properties"]["analysis_run_id"] == str(analysis_run_id) assert feature["properties"]["dataset_id"] == str(dataset_id) assert feature["properties"]["job_id"] == str(detection.job_id) assert feature["properties"]["source_tile_path"] == "storage/tiles/tile_0000.tif" assert feature["properties"]["bbox_json"] == {"x_min": 1, "y_min": 2, "x_max": 10, "y_max": 12} def test_detection_list_filters_by_dataset_class_and_confidence() -> None: project_id = uuid4() dataset_id = uuid4() other_dataset_id = uuid4() analysis_run_id = uuid4() rows = [ _detection(project_id, dataset_id, analysis_run_id, "building", 0.91), _detection(project_id, dataset_id, analysis_run_id, "road", 0.95), _detection(project_id, dataset_id, analysis_run_id, "building", 0.25), _detection(project_id, other_dataset_id, analysis_run_id, "building", 0.99), ] db = FakeSession( objects={(AnalysisRun, analysis_run_id): AnalysisRun(id=analysis_run_id, project_id=project_id, analysis_type="detection", status="success", parameters_json={})}, query_rows={Detection: rows}, ) result = DetectionService.list_detections( db, analysis_run_id=analysis_run_id, dataset_id=dataset_id, class_name="building", min_confidence=0.5, ) assert result.total == 1 assert result.items[0].class_name == "building" assert result.items[0].confidence == 0.91 def test_detection_detail_returns_one_detection() -> None: project_id = uuid4() dataset_id = uuid4() analysis_run_id = uuid4() detection = _detection(project_id, dataset_id, analysis_run_id) db = FakeSession(objects={(Detection, detection.id): detection}) result = DetectionService.get_detection(db, detection.id) assert result.id == detection.id assert result.class_name == "building" def test_detection_models_api_envelope_still_canonical() -> None: response = TestClient(app).get("/api/v1/detection/models") assert response.status_code == 200 assert "data" in response.json() assert "models" in response.json()["data"] def test_detection_geojson_api_uses_canonical_envelope(monkeypatch) -> None: analysis_run_id = uuid4() monkeypatch.setattr( "app.api.routes.detection.DetectionService.detections_to_geojson", lambda *_args, **_kwargs: {"type": "FeatureCollection", "features": []}, ) app.dependency_overrides[get_db] = lambda: FakeSession() try: response = TestClient(app).get(f"/api/v1/detection/runs/{analysis_run_id}/geojson") finally: app.dependency_overrides.pop(get_db, None) assert response.status_code == 200 assert response.json() == {"data": {"type": "FeatureCollection", "features": []}} def test_detection_qa_persists_quality_check_and_metrics() -> None: project_id = uuid4() dataset_id = uuid4() reference_dataset_id = uuid4() analysis_run_id = uuid4() detection = _detection(project_id, dataset_id, analysis_run_id, geom=box(0, 0, 1, 1)) reference_dataset = _authoritative_reference(Dataset( id=reference_dataset_id, project_id=project_id, name="reference.geojson", dataset_type="vector", source="test", dataset_role="reference", )) reference_feature = VectorFeature( id=uuid4(), dataset_id=reference_dataset_id, feature_class="building", geometry=from_shape(box(0, 0, 1, 1), srid=4326), ) db = FakeSession( objects={ (AnalysisRun, analysis_run_id): AnalysisRun(id=analysis_run_id, project_id=project_id, dataset_id=dataset_id, analysis_type="detection", status="success", parameters_json={}), (Dataset, dataset_id): _source_dataset(project_id, dataset_id), (Dataset, reference_dataset_id): reference_dataset, }, query_rows={Detection: [detection], VectorFeature: [reference_feature]}, ) result = DetectionService.compare_detections_with_reference( db=db, analysis_run_id=analysis_run_id, reference_dataset_id=reference_dataset_id, iou_threshold=0.5, ) quality_checks = [item for item in db.added if isinstance(item, QualityCheck)] metrics = [item for item in db.added if isinstance(item, Metric)] assert result["matches"] == 1 assert result["precision"] == 1.0 assert result["recall"] == 1.0 assert result["f1_score"] == 1.0 assert result["quality_check_id"] == str(quality_checks[0].id) assert quality_checks[0].analysis_run_id == analysis_run_id assert quality_checks[0].reference_dataset_id == reference_dataset_id assert quality_checks[0].parameters_json["temporal_compatibility"]["status"] == "compatible" assert quality_checks[0].findings_json["temporal_compatibility"]["status"] == "compatible" assert [metric.metric_key for metric in metrics] == [ "precision", "recall", "f1", "mean_iou", "false_positive_count", "false_negative_count", # Threshold-independent metrics, so two models can be compared without # both having to be read at the same confidence cut. "average_precision", "best_f1", "best_f1_threshold", ] def test_detection_qa_rejects_non_overlapping_historical_reference_editions() -> None: project_id = uuid4() dataset_id = uuid4() reference_dataset_id = uuid4() analysis_run_id = uuid4() source_dataset = _source_dataset(project_id, dataset_id) source_dataset.source_name = "digitaal_vlaanderen_orthophoto" source_dataset.source_metadata = {"product_key": "2020", "supports_detection": False} source_dataset.valid_from = datetime(2020, 1, 1, tzinfo=UTC) source_dataset.valid_to = datetime(2020, 12, 31, 23, 59, 59, tzinfo=UTC) reference_dataset = _authoritative_reference(Dataset( id=reference_dataset_id, project_id=project_id, name="current-grb.geojson", dataset_type="vector", source="grb", source_name="grb", dataset_role="reference", valid_from=datetime(2026, 7, 1, tzinfo=UTC), valid_to=datetime(2026, 7, 31, 23, 59, 59, tzinfo=UTC), )) db = FakeSession( objects={ (AnalysisRun, analysis_run_id): AnalysisRun( id=analysis_run_id, project_id=project_id, dataset_id=dataset_id, analysis_type="detection", status="success", parameters_json={}, ), (Dataset, dataset_id): source_dataset, (Dataset, reference_dataset_id): reference_dataset, }, ) with pytest.raises(AppError) as exc_info: DetectionService.compare_detections_with_reference( db=db, analysis_run_id=analysis_run_id, reference_dataset_id=reference_dataset_id, ) assert exc_info.value.code == "DETECTION_QA_TEMPORAL_MISMATCH" assert db.added == [] def test_detection_qa_no_match_case_persists_zero_scores() -> None: project_id = uuid4() dataset_id = uuid4() reference_dataset_id = uuid4() analysis_run_id = uuid4() detection = _detection(project_id, dataset_id, analysis_run_id, geom=box(0, 0, 1, 1)) reference_dataset = _authoritative_reference(Dataset( id=reference_dataset_id, project_id=project_id, name="reference.geojson", dataset_type="vector", source="test", dataset_role="reference", )) reference_feature = VectorFeature( id=uuid4(), dataset_id=reference_dataset_id, feature_class="building", geometry=from_shape(box(10, 10, 11, 11), srid=4326), ) db = FakeSession( objects={ (AnalysisRun, analysis_run_id): AnalysisRun(id=analysis_run_id, project_id=project_id, dataset_id=dataset_id, analysis_type="detection", status="success", parameters_json={}), (Dataset, dataset_id): _source_dataset(project_id, dataset_id), (Dataset, reference_dataset_id): reference_dataset, }, query_rows={Detection: [detection], VectorFeature: [reference_feature]}, ) result = DetectionService.compare_detections_with_reference( db=db, analysis_run_id=analysis_run_id, reference_dataset_id=reference_dataset_id, iou_threshold=0.5, ) assert result["matches"] == 0 assert result["false_positives"] == 1 assert result["false_negatives"] == 1 assert result["precision"] == 0.0 assert result["recall"] == 0.0 assert result["f1_score"] == 0.0 def test_configured_yolo_qa_requires_persisted_tile_manifest_provenance() -> None: project_id = uuid4() dataset_id = uuid4() reference_dataset_id = uuid4() analysis_run_id = uuid4() detection = _detection(project_id, dataset_id, analysis_run_id) reference_dataset = _authoritative_reference(Dataset( id=reference_dataset_id, project_id=project_id, name="reference.geojson", dataset_type="vector", source="test", dataset_role="reference", )) reference_feature = VectorFeature( id=uuid4(), dataset_id=reference_dataset_id, feature_class="building", geometry=from_shape(box(4.0, 51.0, 4.1, 51.1), srid=4326), ) db = FakeSession( objects={ (AnalysisRun, analysis_run_id): AnalysisRun( id=analysis_run_id, project_id=project_id, dataset_id=dataset_id, analysis_type="detection", status="success", model_name="yolo-configured", parameters_json={"model_id": "yolo-configured"}, ), (Dataset, dataset_id): _source_dataset(project_id, dataset_id), (Dataset, reference_dataset_id): reference_dataset, }, query_rows={Detection: [detection], VectorFeature: [reference_feature]}, ) with pytest.raises(AppError) as exc_info: DetectionService.compare_detections_with_reference( db=db, analysis_run_id=analysis_run_id, reference_dataset_id=reference_dataset_id, iou_threshold=0.5, ) assert exc_info.value.code == "DETECTION_QA_COVERAGE_UNAVAILABLE" assert db.added == [] def _coverage_manifest(tmp_path, dataset_id, bounds=(-1.0, -1.0, 3.0, 3.0)): manifest_path = tmp_path / "manifest.json" manifest_path.write_text( json.dumps( { "source_dataset_id": str(dataset_id), "crs": "EPSG:4326", "tiles": [ { "index": 0, "path": "tile_0000.tif", "bounds": list(bounds), "crs": "EPSG:4326", } ], } ), encoding="utf-8", ) return manifest_path def test_detection_qa_excludes_references_outside_persisted_tile_coverage(tmp_path, monkeypatch) -> None: # QA reads process settings; a manifest written into tmp_path is only a # governed artifact if tmp_path is the storage root. monkeypatch.setenv("STORAGE_ROOT", str(tmp_path)) project_id = uuid4() dataset_id = uuid4() reference_dataset_id = uuid4() analysis_run_id = uuid4() manifest_path = _coverage_manifest(tmp_path, dataset_id, bounds=(0.0, 0.0, 1.0, 1.0)) detection = _detection(project_id, dataset_id, analysis_run_id, geom=box(0.1, 0.1, 0.9, 0.9)) reference_dataset = _authoritative_reference(Dataset( id=reference_dataset_id, project_id=project_id, name="reference.geojson", dataset_type="vector", source="test", dataset_role="reference", )) inside_reference = VectorFeature( id=uuid4(), dataset_id=reference_dataset_id, feature_class="building", geometry=from_shape(box(0.1, 0.1, 0.9, 0.9), srid=4326), ) outside_reference = VectorFeature( id=uuid4(), dataset_id=reference_dataset_id, feature_class="building", geometry=from_shape(box(10.0, 10.0, 11.0, 11.0), srid=4326), ) db = FakeSession( objects={ (AnalysisRun, analysis_run_id): AnalysisRun( id=analysis_run_id, project_id=project_id, dataset_id=dataset_id, analysis_type="detection", status="success", model_name="yolo-configured", parameters_json={ "model_id": "yolo-configured", "tile_manifest_path": str(manifest_path), }, ), (Dataset, dataset_id): _source_dataset(project_id, dataset_id), (Dataset, reference_dataset_id): reference_dataset, }, query_rows={Detection: [detection], VectorFeature: [inside_reference, outside_reference]}, ) result = DetectionService.compare_detections_with_reference( db=db, analysis_run_id=analysis_run_id, reference_dataset_id=reference_dataset_id, iou_threshold=0.5, ) quality_check = next(item for item in db.added if isinstance(item, QualityCheck)) assert result["matches"] == 1 assert result["false_negatives"] == 0 assert result["reference_feature_count_raw"] == 2 assert result["reference_feature_count"] == 1 assert result["coverage"]["applied"] is True assert result["coverage"]["reference_excluded_outside_count"] == 1 assert quality_check.parameters_json["coverage_policy"] == "persisted_tile_manifest_union" assert quality_check.findings_json["coverage"] == result["coverage"] def test_detection_qa_reports_box_to_footprint_diagnostic_without_changing_strict_metrics(tmp_path, monkeypatch) -> None: # QA reads process settings; a manifest written into tmp_path is only a # governed artifact if tmp_path is the storage root. monkeypatch.setenv("STORAGE_ROOT", str(tmp_path)) project_id = uuid4() dataset_id = uuid4() reference_dataset_id = uuid4() analysis_run_id = uuid4() manifest_path = _coverage_manifest(tmp_path, dataset_id) detection = _detection(project_id, dataset_id, analysis_run_id, geom=box(0.0, 0.0, 2.0, 2.0)) l_shaped_footprint = Polygon( [(0.0, 0.0), (2.0, 0.0), (2.0, 0.4), (0.4, 0.4), (0.4, 2.0), (0.0, 2.0), (0.0, 0.0)] ) reference_dataset = _authoritative_reference(Dataset( id=reference_dataset_id, project_id=project_id, name="reference.geojson", dataset_type="vector", source="test", dataset_role="reference", )) reference_feature = VectorFeature( id=uuid4(), dataset_id=reference_dataset_id, feature_class="building", geometry=from_shape(l_shaped_footprint, srid=4326), ) db = FakeSession( objects={ (AnalysisRun, analysis_run_id): AnalysisRun( id=analysis_run_id, project_id=project_id, dataset_id=dataset_id, analysis_type="detection", status="success", model_name="yolo-configured", parameters_json={ "model_id": "yolo-configured", "tile_manifest_path": str(manifest_path), }, ), (Dataset, dataset_id): _source_dataset(project_id, dataset_id), (Dataset, reference_dataset_id): reference_dataset, }, query_rows={Detection: [detection], VectorFeature: [reference_feature]}, ) result = DetectionService.compare_detections_with_reference( db=db, analysis_run_id=analysis_run_id, reference_dataset_id=reference_dataset_id, iou_threshold=0.5, ) diagnostics = result["box_to_footprint_diagnostics"] quality_check = next(item for item in db.added if isinstance(item, QualityCheck)) assert result["matches"] == 0 assert result["false_positives"] == 1 assert result["false_negatives"] == 1 assert diagnostics["diagnostic_only"] is True assert diagnostics["envelope_matches"] == 1 assert diagnostics["possible_box_to_footprint_mismatch_count"] == 1 assert quality_check.findings_json["box_to_footprint_diagnostics"] == diagnostics