Initial public release

backend/tests/test_sprint176_detection_false_positive_visual_review.py

@@ -0,0 +1,446 @@
+from __future__ import annotations
+
+import csv
+import json
+import subprocess
+import sys
+from pathlib import Path
+from uuid import uuid4
+
+import numpy as np
+import rasterio
+from geoalchemy2.shape import from_shape
+from PIL import Image
+from rasterio.transform import from_bounds
+from shapely.geometry import box, mapping
+
+from app.models import Detection, QualityCheck
+from app.services.quality_evidence_service import QualityEvidenceService
+
+
+ROOT = Path(__file__).resolve().parents[2]
+
+
+class FakeQuery:
+    def __init__(self, rows):
+        self.rows = list(rows)
+
+    def filter(self, *criteria):
+        for criterion in criteria:
+            left = getattr(criterion, "left", None)
+            right = getattr(criterion, "right", None)
+            operator = getattr(criterion, "operator", None)
+            name = getattr(left, "name", None)
+            value = getattr(right, "value", right)
+            if name and operator and operator.__name__ == "eq":
+                self.rows = [row for row in self.rows if getattr(row, name) == value]
+        return self
+
+    def all(self):
+        return list(self.rows)
+
+
+class FakeSession:
+    def __init__(self, objects=None, query_rows=None) -> None:
+        self.objects = objects or {}
+        self.query_rows = query_rows or {}
+
+    def get(self, model, item_id):
+        return self.objects.get((model, item_id))
+
+    def query(self, model):
+        return FakeQuery(self.query_rows.get(model, []))
+
+
+def test_detection_quality_evidence_exposes_persisted_detection_provenance() -> None:
+    project_id = uuid4()
+    dataset_id = uuid4()
+    reference_dataset_id = uuid4()
+    analysis_run_id = uuid4()
+    quality_check_id = uuid4()
+    detection = Detection(
+        id=uuid4(),
+        project_id=project_id,
+        dataset_id=dataset_id,
+        analysis_run_id=analysis_run_id,
+        job_id=uuid4(),
+        model_name="yolo-configured",
+        model_version="review-model",
+        class_name="building",
+        confidence=0.73,
+        geometry=from_shape(box(5.0, 51.0, 5.001, 51.001), srid=4326),
+        bbox_json={"x_min": 12.0, "y_min": 18.0, "x_max": 42.0, "y_max": 51.0},
+        source_tile_path="/app/storage/tiles/review/tile_0003.tif",
+        properties_json={"class_id": 0, "tile_index": 3},
+    )
+    quality_check = QualityCheck(
+        id=quality_check_id,
+        project_id=project_id,
+        analysis_run_id=analysis_run_id,
+        candidate_dataset_id=dataset_id,
+        reference_dataset_id=reference_dataset_id,
+        check_type="detections_vs_reference",
+        status="ok",
+        findings_json={
+            "false_positive_evidence": [{"candidate_feature_id": str(detection.id)}]
+        },
+    )
+    db = FakeSession(
+        objects={(QualityCheck, quality_check_id): quality_check},
+        query_rows={Detection: [detection]},
+    )
+
+    result = QualityEvidenceService.evidence_geojson(
+        db,
+        project_id=project_id,
+        quality_check_id=quality_check_id,
+    )
+
+    properties = result["geojson"]["features"][0]["properties"]
+    assert properties["qa_evidence_role"] == "false_positive"
+    assert properties["detection_id"] == str(detection.id)
+    assert properties["confidence"] == 0.73
+    assert properties["model_name"] == "yolo-configured"
+    assert properties["model_version"] == "review-model"
+    assert properties["source_tile_path"] == "/app/storage/tiles/review/tile_0003.tif"
+    assert properties["bbox_json"] == {
+        "x_min": 12.0,
+        "y_min": 18.0,
+        "x_max": 42.0,
+        "y_max": 51.0,
+    }
+    assert properties["tile_index"] == 3
+
+
+def _write_geotiff(path: Path, *, seed: int, width: int = 128, height: int = 128) -> None:
+    rng = np.random.default_rng(seed)
+    data = rng.integers(35, 190, size=(3, height, width), dtype=np.uint8)
+    data[:, 32:92, 38:98] = np.array([190, 180, 165], dtype=np.uint8)[:, None, None]
+    path.parent.mkdir(parents=True, exist_ok=True)
+    with rasterio.open(
+        path,
+        "w",
+        driver="GTiff",
+        width=width,
+        height=height,
+        count=3,
+        dtype="uint8",
+        crs="EPSG:4326",
+        transform=from_bounds(5.0, 51.0, 5.01, 51.01, width, height),
+    ) as dataset:
+        dataset.write(data)
+
+
+def _feature(
+    role: str,
+    feature_id: str,
+    geometry: dict,
+    *,
+    tile_path: Path | None = None,
+    confidence: float | None = None,
+    bbox: dict | None = None,
+) -> dict:
+    properties = {
+        "qa_evidence_role": role,
+        "feature_id": feature_id,
+        "candidate_feature_id": feature_id if role in {"false_positive", "match_candidate"} else None,
+        "reference_feature_id": feature_id if role in {"false_negative", "match_reference"} else None,
+        "analysis_run_id": "run-review",
+        "quality_check_id": "quality-review",
+        "feature_class": "building",
+    }
+    if tile_path is not None:
+        properties.update(
+            {
+                "detection_id": feature_id,
+                "confidence": confidence,
+                "model_name": "yolo-configured",
+                "model_version": "review-model",
+                "source_tile_path": str(tile_path),
+                "bbox_json": bbox,
+                "tile_index": 0,
+            }
+        )
+    return {
+        "type": "Feature",
+        "id": f"{role}:{feature_id}",
+        "properties": properties,
+        "geometry": geometry,
+    }
+
+
+def _write_review_portfolio(tmp_path: Path, *, unsafe_tile: bool = False) -> tuple[Path, Path]:
+    storage_root = tmp_path / "storage"
+    samples = []
+    for sample_index, sample_slug in enumerate(("geel", "turnhout")):
+        tile_path = storage_root / sample_slug / "tile_0000.tif"
+        _write_geotiff(
+            tile_path,
+            seed=sample_index + 1,
+            height=80 if sample_slug == "turnhout" else 128,
+        )
+        selected_tile = (tmp_path / "outside.tif") if unsafe_tile and sample_slug == "geel" else tile_path
+        if unsafe_tile and sample_slug == "geel":
+            _write_geotiff(selected_tile, seed=99)
+        features = [
+            _feature(
+                "false_positive",
+                f"{sample_slug}-low-small",
+                mapping(box(5.001, 51.001, 5.0014, 51.0014)),
+                tile_path=selected_tile,
+                confidence=0.22,
+                bbox={"x_min": 18, "y_min": 22, "x_max": 35, "y_max": 39},
+            ),
+            _feature(
+                "false_positive",
+                f"{sample_slug}-mid-medium",
+                mapping(box(5.003, 51.003, 5.004, 51.004)),
+                tile_path=tile_path,
+                confidence=0.48,
+                bbox={"x_min": 45, "y_min": 48, "x_max": 76, "y_max": 79},
+            ),
+            _feature(
+                "false_positive",
+                f"{sample_slug}-high-large",
+                mapping(box(5.005, 51.005, 5.007, 51.007)),
+                tile_path=tile_path,
+                confidence=0.81,
+                bbox={"x_min": 70, "y_min": 18, "x_max": 111, "y_max": 62},
+            ),
+            _feature(
+                "match_reference",
+                f"{sample_slug}-reference",
+                mapping(box(5.002, 51.002, 5.003, 51.003)),
+            ),
+            _feature(
+                "false_negative",
+                f"{sample_slug}-missed-reference",
+                mapping(box(5.006, 51.002, 5.007, 51.003)),
+            ),
+        ]
+        evidence_dir = tmp_path / "portfolio" / "samples" / sample_slug / "evidence"
+        evidence_dir.mkdir(parents=True)
+        evidence_path = evidence_dir / "calibration_evidence.geojson"
+        evidence_path.write_text(
+            json.dumps({"type": "FeatureCollection", "features": features}),
+            encoding="utf-8",
+        )
+        samples.append(
+            {
+                "sample_slug": sample_slug,
+                "aoi_label": sample_slug.title(),
+                "role_counts": {
+                    "false_positive": 3,
+                    "match_reference": 1,
+                    "false_negative": 1,
+                },
+                "evidence_geojson_path": str(evidence_path),
+            }
+        )
+    portfolio_path = tmp_path / "portfolio" / "calibration_evidence_portfolio.json"
+    portfolio_path.write_text(
+        json.dumps(
+            {
+                "model_asset_id": "model-review",
+                "model_sha256": "abc123",
+                "samples": samples,
+            }
+        ),
+        encoding="utf-8",
+    )
+    return portfolio_path, storage_root
+
+
+def test_false_positive_visual_review_is_stratified_and_requires_manual_decisions(
+    tmp_path: Path,
+) -> None:
+    renderer = ROOT / "scripts" / "render_detection_false_positive_review_contact_sheets.py"
+    validator = ROOT / "scripts" / "validate_detection_false_positive_review_decisions.py"
+    readiness = (ROOT / "scripts" / "run_readiness_check.sh").read_text(encoding="utf-8")
+    dockerfile = (ROOT / "deploy" / "unraid" / "Dockerfile.all-in-one").read_text(
+        encoding="utf-8"
+    )
+    assert renderer.exists()
+    assert validator.exists()
+    assert "py_compile scripts/render_detection_false_positive_review_contact_sheets.py" in readiness
+    assert "py_compile scripts/validate_detection_false_positive_review_decisions.py" in readiness
+    assert "COPY scripts/render_detection_false_positive_review_contact_sheets.py" in dockerfile
+    assert "COPY scripts/validate_detection_false_positive_review_decisions.py" in dockerfile
+
+    portfolio_path, storage_root = _write_review_portfolio(tmp_path)
+    output_dir = tmp_path / "review"
+    result = subprocess.run(
+        [
+            sys.executable,
+            str(renderer),
+            "--portfolio",
+            str(portfolio_path),
+            "--storage-root",
+            str(storage_root),
+            "--output-dir",
+            str(output_dir),
+            "--sample-slugs",
+            "geel,turnhout",
+            "--max-features",
+            "4",
+            "--columns",
+            "2",
+            "--cards-per-sheet",
+            "4",
+            "--thumb-size",
+            "128",
+        ],
+        cwd=ROOT,
+        check=True,
+        capture_output=True,
+        text=True,
+    )
+
+    report = json.loads(
+        (output_dir / "detection_false_positive_review_summary.json").read_text(
+            encoding="utf-8"
+        )
+    )
+    assert report["status"] == "review_required"
+    assert report["population_count"] == 6
+    assert report["selected_feature_count"] == 4
+    assert report["selected_sample_slugs"] == ["geel", "turnhout"]
+    assert report["missing_provenance_count"] == 0
+    assert report["missing_tile_count"] == 0
+    assert report["reference_overlay_feature_count"] > 0
+    assert set(report["selected_area_buckets"])
+    assert set(report["selected_confidence_bands"])
+    assert "review required" in result.stdout.lower()
+
+    sheet_path = output_dir / report["contact_sheets"][0]["path"]
+    sheet = Image.open(sheet_path).convert("RGB")
+    assert sheet.width >= 256
+    assert sheet.height >= 256
+    assert len(sheet.getcolors(maxcolors=1_000_000) or []) > 20
+
+    decisions_path = output_dir / "false_positive_review_decisions.csv"
+    with decisions_path.open(newline="", encoding="utf-8") as handle:
+        rows = list(csv.DictReader(handle))
+    assert len(rows) == 4
+    assert {row["review_decision"] for row in rows} == {"unreviewed"}
+    assert all(row["candidate_feature_id"] for row in rows)
+    assert all(row["source_tile_path"] for row in rows)
+
+    incomplete_dir = tmp_path / "incomplete"
+    incomplete = subprocess.run(
+        [
+            sys.executable,
+            str(validator),
+            "--review-summary",
+            str(output_dir / "detection_false_positive_review_summary.json"),
+            "--decisions-csv",
+            str(decisions_path),
+            "--output-dir",
+            str(incomplete_dir),
+            "--require-complete",
+        ],
+        cwd=ROOT,
+        check=False,
+        capture_output=True,
+        text=True,
+    )
+    assert incomplete.returncode == 2
+    incomplete_validation = json.loads(
+        (incomplete_dir / "detection_false_positive_review_validation.json").read_text(
+            encoding="utf-8"
+        )
+    )
+    assert incomplete_validation["status"] == "review_required"
+    assert incomplete_validation["decision_counts"]["unreviewed"] == 4
+    incomplete_confirmed = json.loads(
+        (incomplete_dir / "confirmed_model_false_positives.geojson").read_text(
+            encoding="utf-8"
+        )
+    )
+    assert incomplete_confirmed["features"] == []
+
+    decisions = (
+        "confirmed_model_false_positive",
+        "reference_gap_or_change",
+        "qa_alignment_mismatch",
+        "uncertain",
+    )
+    for row, decision in zip(rows, decisions, strict=True):
+        row["review_decision"] = decision
+        row["review_notes"] = f"reviewed as {decision}"
+    with decisions_path.open("w", newline="", encoding="utf-8") as handle:
+        writer = csv.DictWriter(handle, fieldnames=list(rows[0]))
+        writer.writeheader()
+        writer.writerows(rows)
+
+    validation_dir = tmp_path / "validated"
+    subprocess.run(
+        [
+            sys.executable,
+            str(validator),
+            "--review-summary",
+            str(output_dir / "detection_false_positive_review_summary.json"),
+            "--decisions-csv",
+            str(decisions_path),
+            "--output-dir",
+            str(validation_dir),
+            "--require-complete",
+        ],
+        cwd=ROOT,
+        check=True,
+        capture_output=True,
+        text=True,
+    )
+    validation = json.loads(
+        (validation_dir / "detection_false_positive_review_validation.json").read_text(
+            encoding="utf-8"
+        )
+    )
+    assert validation["status"] == "complete"
+    assert validation["decision_counts"] == {
+        "confirmed_model_false_positive": 1,
+        "qa_alignment_mismatch": 1,
+        "reference_gap_or_change": 1,
+        "uncertain": 1,
+        "unreviewed": 0,
+    }
+    confirmed = json.loads(
+        (validation_dir / "confirmed_model_false_positives.geojson").read_text(
+            encoding="utf-8"
+        )
+    )
+    assert len(confirmed["features"]) == 1
+    assert confirmed["features"][0]["properties"]["review_decision"] == (
+        "confirmed_model_false_positive"
+    )
+
+
+def test_false_positive_visual_review_rejects_tiles_outside_storage_root(
+    tmp_path: Path,
+) -> None:
+    renderer = ROOT / "scripts" / "render_detection_false_positive_review_contact_sheets.py"
+    portfolio_path, storage_root = _write_review_portfolio(tmp_path, unsafe_tile=True)
+
+    result = subprocess.run(
+        [
+            sys.executable,
+            str(renderer),
+            "--portfolio",
+            str(portfolio_path),
+            "--storage-root",
+            str(storage_root),
+            "--output-dir",
+            str(tmp_path / "review"),
+            "--sample-slugs",
+            "geel",
+            "--max-features",
+            "3",
+        ],
+        cwd=ROOT,
+        check=False,
+        capture_output=True,
+        text=True,
+    )
+
+    assert result.returncode != 0
+    assert "outside storage root" in result.stderr