from __future__ import annotations import csv import json import subprocess import sys from pathlib import Path from uuid import uuid4 import numpy as np import rasterio from geoalchemy2.shape import from_shape from PIL import Image from rasterio.transform import from_bounds from shapely.geometry import box, mapping from app.models import Detection, QualityCheck from app.services.quality_evidence_service import QualityEvidenceService ROOT = Path(__file__).resolve().parents[2] class FakeQuery: def __init__(self, rows): self.rows = list(rows) def filter(self, *criteria): for criterion in criteria: left = getattr(criterion, "left", None) right = getattr(criterion, "right", None) operator = getattr(criterion, "operator", None) name = getattr(left, "name", None) value = getattr(right, "value", right) if name and operator and operator.__name__ == "eq": self.rows = [row for row in self.rows if getattr(row, name) == value] return self def all(self): return list(self.rows) class FakeSession: def __init__(self, objects=None, query_rows=None) -> None: self.objects = objects or {} self.query_rows = query_rows or {} def get(self, model, item_id): return self.objects.get((model, item_id)) def query(self, model): return FakeQuery(self.query_rows.get(model, [])) def test_detection_quality_evidence_exposes_persisted_detection_provenance() -> None: project_id = uuid4() dataset_id = uuid4() reference_dataset_id = uuid4() analysis_run_id = uuid4() quality_check_id = uuid4() detection = Detection( id=uuid4(), project_id=project_id, dataset_id=dataset_id, analysis_run_id=analysis_run_id, job_id=uuid4(), model_name="yolo-configured", model_version="review-model", class_name="building", confidence=0.73, geometry=from_shape(box(5.0, 51.0, 5.001, 51.001), srid=4326), bbox_json={"x_min": 12.0, "y_min": 18.0, "x_max": 42.0, "y_max": 51.0}, source_tile_path="/app/storage/tiles/review/tile_0003.tif", properties_json={"class_id": 0, "tile_index": 3}, ) quality_check = QualityCheck( id=quality_check_id, project_id=project_id, analysis_run_id=analysis_run_id, candidate_dataset_id=dataset_id, reference_dataset_id=reference_dataset_id, check_type="detections_vs_reference", status="ok", findings_json={ "false_positive_evidence": [{"candidate_feature_id": str(detection.id)}] }, ) db = FakeSession( objects={(QualityCheck, quality_check_id): quality_check}, query_rows={Detection: [detection]}, ) result = QualityEvidenceService.evidence_geojson( db, project_id=project_id, quality_check_id=quality_check_id, ) properties = result["geojson"]["features"][0]["properties"] assert properties["qa_evidence_role"] == "false_positive" assert properties["detection_id"] == str(detection.id) assert properties["confidence"] == 0.73 assert properties["model_name"] == "yolo-configured" assert properties["model_version"] == "review-model" assert properties["source_tile_path"] == "/app/storage/tiles/review/tile_0003.tif" assert properties["bbox_json"] == { "x_min": 12.0, "y_min": 18.0, "x_max": 42.0, "y_max": 51.0, } assert properties["tile_index"] == 3 def _write_geotiff(path: Path, *, seed: int, width: int = 128, height: int = 128) -> None: rng = np.random.default_rng(seed) data = rng.integers(35, 190, size=(3, height, width), dtype=np.uint8) data[:, 32:92, 38:98] = np.array([190, 180, 165], dtype=np.uint8)[:, None, None] path.parent.mkdir(parents=True, exist_ok=True) with rasterio.open( path, "w", driver="GTiff", width=width, height=height, count=3, dtype="uint8", crs="EPSG:4326", transform=from_bounds(5.0, 51.0, 5.01, 51.01, width, height), ) as dataset: dataset.write(data) def _feature( role: str, feature_id: str, geometry: dict, *, tile_path: Path | None = None, confidence: float | None = None, bbox: dict | None = None, ) -> dict: properties = { "qa_evidence_role": role, "feature_id": feature_id, "candidate_feature_id": feature_id if role in {"false_positive", "match_candidate"} else None, "reference_feature_id": feature_id if role in {"false_negative", "match_reference"} else None, "analysis_run_id": "run-review", "quality_check_id": "quality-review", "feature_class": "building", } if tile_path is not None: properties.update( { "detection_id": feature_id, "confidence": confidence, "model_name": "yolo-configured", "model_version": "review-model", "source_tile_path": str(tile_path), "bbox_json": bbox, "tile_index": 0, } ) return { "type": "Feature", "id": f"{role}:{feature_id}", "properties": properties, "geometry": geometry, } def _write_review_portfolio(tmp_path: Path, *, unsafe_tile: bool = False) -> tuple[Path, Path]: storage_root = tmp_path / "storage" samples = [] for sample_index, sample_slug in enumerate(("geel", "turnhout")): tile_path = storage_root / sample_slug / "tile_0000.tif" _write_geotiff( tile_path, seed=sample_index + 1, height=80 if sample_slug == "turnhout" else 128, ) selected_tile = (tmp_path / "outside.tif") if unsafe_tile and sample_slug == "geel" else tile_path if unsafe_tile and sample_slug == "geel": _write_geotiff(selected_tile, seed=99) features = [ _feature( "false_positive", f"{sample_slug}-low-small", mapping(box(5.001, 51.001, 5.0014, 51.0014)), tile_path=selected_tile, confidence=0.22, bbox={"x_min": 18, "y_min": 22, "x_max": 35, "y_max": 39}, ), _feature( "false_positive", f"{sample_slug}-mid-medium", mapping(box(5.003, 51.003, 5.004, 51.004)), tile_path=tile_path, confidence=0.48, bbox={"x_min": 45, "y_min": 48, "x_max": 76, "y_max": 79}, ), _feature( "false_positive", f"{sample_slug}-high-large", mapping(box(5.005, 51.005, 5.007, 51.007)), tile_path=tile_path, confidence=0.81, bbox={"x_min": 70, "y_min": 18, "x_max": 111, "y_max": 62}, ), _feature( "match_reference", f"{sample_slug}-reference", mapping(box(5.002, 51.002, 5.003, 51.003)), ), _feature( "false_negative", f"{sample_slug}-missed-reference", mapping(box(5.006, 51.002, 5.007, 51.003)), ), ] evidence_dir = tmp_path / "portfolio" / "samples" / sample_slug / "evidence" evidence_dir.mkdir(parents=True) evidence_path = evidence_dir / "calibration_evidence.geojson" evidence_path.write_text( json.dumps({"type": "FeatureCollection", "features": features}), encoding="utf-8", ) samples.append( { "sample_slug": sample_slug, "aoi_label": sample_slug.title(), "role_counts": { "false_positive": 3, "match_reference": 1, "false_negative": 1, }, "evidence_geojson_path": str(evidence_path), } ) portfolio_path = tmp_path / "portfolio" / "calibration_evidence_portfolio.json" portfolio_path.write_text( json.dumps( { "model_asset_id": "model-review", "model_sha256": "abc123", "samples": samples, } ), encoding="utf-8", ) return portfolio_path, storage_root def test_false_positive_visual_review_is_stratified_and_requires_manual_decisions( tmp_path: Path, ) -> None: renderer = ROOT / "scripts" / "render_detection_false_positive_review_contact_sheets.py" validator = ROOT / "scripts" / "validate_detection_false_positive_review_decisions.py" readiness = (ROOT / "scripts" / "run_readiness_check.sh").read_text(encoding="utf-8") dockerfile = (ROOT / "deploy" / "unraid" / "Dockerfile.all-in-one").read_text( encoding="utf-8" ) assert renderer.exists() assert validator.exists() assert "py_compile scripts/render_detection_false_positive_review_contact_sheets.py" in readiness assert "py_compile scripts/validate_detection_false_positive_review_decisions.py" in readiness assert "COPY scripts/render_detection_false_positive_review_contact_sheets.py" in dockerfile assert "COPY scripts/validate_detection_false_positive_review_decisions.py" in dockerfile portfolio_path, storage_root = _write_review_portfolio(tmp_path) output_dir = tmp_path / "review" result = subprocess.run( [ sys.executable, str(renderer), "--portfolio", str(portfolio_path), "--storage-root", str(storage_root), "--output-dir", str(output_dir), "--sample-slugs", "geel,turnhout", "--max-features", "4", "--columns", "2", "--cards-per-sheet", "4", "--thumb-size", "128", ], cwd=ROOT, check=True, capture_output=True, text=True, ) report = json.loads( (output_dir / "detection_false_positive_review_summary.json").read_text( encoding="utf-8" ) ) assert report["status"] == "review_required" assert report["population_count"] == 6 assert report["selected_feature_count"] == 4 assert report["selected_sample_slugs"] == ["geel", "turnhout"] assert report["missing_provenance_count"] == 0 assert report["missing_tile_count"] == 0 assert report["reference_overlay_feature_count"] > 0 assert set(report["selected_area_buckets"]) assert set(report["selected_confidence_bands"]) assert "review required" in result.stdout.lower() sheet_path = output_dir / report["contact_sheets"][0]["path"] sheet = Image.open(sheet_path).convert("RGB") assert sheet.width >= 256 assert sheet.height >= 256 assert len(sheet.getcolors(maxcolors=1_000_000) or []) > 20 decisions_path = output_dir / "false_positive_review_decisions.csv" with decisions_path.open(newline="", encoding="utf-8") as handle: rows = list(csv.DictReader(handle)) assert len(rows) == 4 assert {row["review_decision"] for row in rows} == {"unreviewed"} assert all(row["candidate_feature_id"] for row in rows) assert all(row["source_tile_path"] for row in rows) incomplete_dir = tmp_path / "incomplete" incomplete = subprocess.run( [ sys.executable, str(validator), "--review-summary", str(output_dir / "detection_false_positive_review_summary.json"), "--decisions-csv", str(decisions_path), "--output-dir", str(incomplete_dir), "--require-complete", ], cwd=ROOT, check=False, capture_output=True, text=True, ) assert incomplete.returncode == 2 incomplete_validation = json.loads( (incomplete_dir / "detection_false_positive_review_validation.json").read_text( encoding="utf-8" ) ) assert incomplete_validation["status"] == "review_required" assert incomplete_validation["decision_counts"]["unreviewed"] == 4 incomplete_confirmed = json.loads( (incomplete_dir / "confirmed_model_false_positives.geojson").read_text( encoding="utf-8" ) ) assert incomplete_confirmed["features"] == [] decisions = ( "confirmed_model_false_positive", "reference_gap_or_change", "qa_alignment_mismatch", "uncertain", ) for row, decision in zip(rows, decisions, strict=True): row["review_decision"] = decision row["review_notes"] = f"reviewed as {decision}" with decisions_path.open("w", newline="", encoding="utf-8") as handle: writer = csv.DictWriter(handle, fieldnames=list(rows[0])) writer.writeheader() writer.writerows(rows) validation_dir = tmp_path / "validated" subprocess.run( [ sys.executable, str(validator), "--review-summary", str(output_dir / "detection_false_positive_review_summary.json"), "--decisions-csv", str(decisions_path), "--output-dir", str(validation_dir), "--require-complete", ], cwd=ROOT, check=True, capture_output=True, text=True, ) validation = json.loads( (validation_dir / "detection_false_positive_review_validation.json").read_text( encoding="utf-8" ) ) assert validation["status"] == "complete" assert validation["decision_counts"] == { "confirmed_model_false_positive": 1, "qa_alignment_mismatch": 1, "reference_gap_or_change": 1, "uncertain": 1, "unreviewed": 0, } confirmed = json.loads( (validation_dir / "confirmed_model_false_positives.geojson").read_text( encoding="utf-8" ) ) assert len(confirmed["features"]) == 1 assert confirmed["features"][0]["properties"]["review_decision"] == ( "confirmed_model_false_positive" ) def test_false_positive_visual_review_rejects_tiles_outside_storage_root( tmp_path: Path, ) -> None: renderer = ROOT / "scripts" / "render_detection_false_positive_review_contact_sheets.py" portfolio_path, storage_root = _write_review_portfolio(tmp_path, unsafe_tile=True) result = subprocess.run( [ sys.executable, str(renderer), "--portfolio", str(portfolio_path), "--storage-root", str(storage_root), "--output-dir", str(tmp_path / "review"), "--sample-slugs", "geel", "--max-features", "3", ], cwd=ROOT, check=False, capture_output=True, text=True, ) assert result.returncode != 0 assert "outside storage root" in result.stderr