Jens
faeb58ef6d
GeoIntel release gates / Compile, test, contracts and builds (push) Successful in 1m49s
GeoIntel release gates / Python and npm vulnerability policy (push) Successful in 21s
GeoIntel release gates / Production AI image, SBOM and container scan (push) Successful in 5m39s
GeoIntel release gates / Deploy exact gated revision to Unraid (push) Failing after 58m43s
447 lines
15 KiB
Python
447 lines
15 KiB
Python
from __future__ import annotations
|
|
|
|
import csv
|
|
import json
|
|
import subprocess
|
|
import sys
|
|
from pathlib import Path
|
|
from uuid import uuid4
|
|
|
|
import numpy as np
|
|
import rasterio
|
|
from geoalchemy2.shape import from_shape
|
|
from PIL import Image
|
|
from rasterio.transform import from_bounds
|
|
from shapely.geometry import box, mapping
|
|
|
|
from app.models import Detection, QualityCheck
|
|
from app.services.quality_evidence_service import QualityEvidenceService
|
|
|
|
|
|
ROOT = Path(__file__).resolve().parents[2]
|
|
|
|
|
|
class FakeQuery:
|
|
def __init__(self, rows):
|
|
self.rows = list(rows)
|
|
|
|
def filter(self, *criteria):
|
|
for criterion in criteria:
|
|
left = getattr(criterion, "left", None)
|
|
right = getattr(criterion, "right", None)
|
|
operator = getattr(criterion, "operator", None)
|
|
name = getattr(left, "name", None)
|
|
value = getattr(right, "value", right)
|
|
if name and operator and operator.__name__ == "eq":
|
|
self.rows = [row for row in self.rows if getattr(row, name) == value]
|
|
return self
|
|
|
|
def all(self):
|
|
return list(self.rows)
|
|
|
|
|
|
class FakeSession:
|
|
def __init__(self, objects=None, query_rows=None) -> None:
|
|
self.objects = objects or {}
|
|
self.query_rows = query_rows or {}
|
|
|
|
def get(self, model, item_id):
|
|
return self.objects.get((model, item_id))
|
|
|
|
def query(self, model):
|
|
return FakeQuery(self.query_rows.get(model, []))
|
|
|
|
|
|
def test_detection_quality_evidence_exposes_persisted_detection_provenance() -> None:
|
|
project_id = uuid4()
|
|
dataset_id = uuid4()
|
|
reference_dataset_id = uuid4()
|
|
analysis_run_id = uuid4()
|
|
quality_check_id = uuid4()
|
|
detection = Detection(
|
|
id=uuid4(),
|
|
project_id=project_id,
|
|
dataset_id=dataset_id,
|
|
analysis_run_id=analysis_run_id,
|
|
job_id=uuid4(),
|
|
model_name="yolo-configured",
|
|
model_version="review-model",
|
|
class_name="building",
|
|
confidence=0.73,
|
|
geometry=from_shape(box(5.0, 51.0, 5.001, 51.001), srid=4326),
|
|
bbox_json={"x_min": 12.0, "y_min": 18.0, "x_max": 42.0, "y_max": 51.0},
|
|
source_tile_path="/app/storage/tiles/review/tile_0003.tif",
|
|
properties_json={"class_id": 0, "tile_index": 3},
|
|
)
|
|
quality_check = QualityCheck(
|
|
id=quality_check_id,
|
|
project_id=project_id,
|
|
analysis_run_id=analysis_run_id,
|
|
candidate_dataset_id=dataset_id,
|
|
reference_dataset_id=reference_dataset_id,
|
|
check_type="detections_vs_reference",
|
|
status="ok",
|
|
findings_json={
|
|
"false_positive_evidence": [{"candidate_feature_id": str(detection.id)}]
|
|
},
|
|
)
|
|
db = FakeSession(
|
|
objects={(QualityCheck, quality_check_id): quality_check},
|
|
query_rows={Detection: [detection]},
|
|
)
|
|
|
|
result = QualityEvidenceService.evidence_geojson(
|
|
db,
|
|
project_id=project_id,
|
|
quality_check_id=quality_check_id,
|
|
)
|
|
|
|
properties = result["geojson"]["features"][0]["properties"]
|
|
assert properties["qa_evidence_role"] == "false_positive"
|
|
assert properties["detection_id"] == str(detection.id)
|
|
assert properties["confidence"] == 0.73
|
|
assert properties["model_name"] == "yolo-configured"
|
|
assert properties["model_version"] == "review-model"
|
|
assert properties["source_tile_path"] == "/app/storage/tiles/review/tile_0003.tif"
|
|
assert properties["bbox_json"] == {
|
|
"x_min": 12.0,
|
|
"y_min": 18.0,
|
|
"x_max": 42.0,
|
|
"y_max": 51.0,
|
|
}
|
|
assert properties["tile_index"] == 3
|
|
|
|
|
|
def _write_geotiff(path: Path, *, seed: int, width: int = 128, height: int = 128) -> None:
|
|
rng = np.random.default_rng(seed)
|
|
data = rng.integers(35, 190, size=(3, height, width), dtype=np.uint8)
|
|
data[:, 32:92, 38:98] = np.array([190, 180, 165], dtype=np.uint8)[:, None, None]
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
with rasterio.open(
|
|
path,
|
|
"w",
|
|
driver="GTiff",
|
|
width=width,
|
|
height=height,
|
|
count=3,
|
|
dtype="uint8",
|
|
crs="EPSG:4326",
|
|
transform=from_bounds(5.0, 51.0, 5.01, 51.01, width, height),
|
|
) as dataset:
|
|
dataset.write(data)
|
|
|
|
|
|
def _feature(
|
|
role: str,
|
|
feature_id: str,
|
|
geometry: dict,
|
|
*,
|
|
tile_path: Path | None = None,
|
|
confidence: float | None = None,
|
|
bbox: dict | None = None,
|
|
) -> dict:
|
|
properties = {
|
|
"qa_evidence_role": role,
|
|
"feature_id": feature_id,
|
|
"candidate_feature_id": feature_id if role in {"false_positive", "match_candidate"} else None,
|
|
"reference_feature_id": feature_id if role in {"false_negative", "match_reference"} else None,
|
|
"analysis_run_id": "run-review",
|
|
"quality_check_id": "quality-review",
|
|
"feature_class": "building",
|
|
}
|
|
if tile_path is not None:
|
|
properties.update(
|
|
{
|
|
"detection_id": feature_id,
|
|
"confidence": confidence,
|
|
"model_name": "yolo-configured",
|
|
"model_version": "review-model",
|
|
"source_tile_path": str(tile_path),
|
|
"bbox_json": bbox,
|
|
"tile_index": 0,
|
|
}
|
|
)
|
|
return {
|
|
"type": "Feature",
|
|
"id": f"{role}:{feature_id}",
|
|
"properties": properties,
|
|
"geometry": geometry,
|
|
}
|
|
|
|
|
|
def _write_review_portfolio(tmp_path: Path, *, unsafe_tile: bool = False) -> tuple[Path, Path]:
|
|
storage_root = tmp_path / "storage"
|
|
samples = []
|
|
for sample_index, sample_slug in enumerate(("geel", "turnhout")):
|
|
tile_path = storage_root / sample_slug / "tile_0000.tif"
|
|
_write_geotiff(
|
|
tile_path,
|
|
seed=sample_index + 1,
|
|
height=80 if sample_slug == "turnhout" else 128,
|
|
)
|
|
selected_tile = (tmp_path / "outside.tif") if unsafe_tile and sample_slug == "geel" else tile_path
|
|
if unsafe_tile and sample_slug == "geel":
|
|
_write_geotiff(selected_tile, seed=99)
|
|
features = [
|
|
_feature(
|
|
"false_positive",
|
|
f"{sample_slug}-low-small",
|
|
mapping(box(5.001, 51.001, 5.0014, 51.0014)),
|
|
tile_path=selected_tile,
|
|
confidence=0.22,
|
|
bbox={"x_min": 18, "y_min": 22, "x_max": 35, "y_max": 39},
|
|
),
|
|
_feature(
|
|
"false_positive",
|
|
f"{sample_slug}-mid-medium",
|
|
mapping(box(5.003, 51.003, 5.004, 51.004)),
|
|
tile_path=tile_path,
|
|
confidence=0.48,
|
|
bbox={"x_min": 45, "y_min": 48, "x_max": 76, "y_max": 79},
|
|
),
|
|
_feature(
|
|
"false_positive",
|
|
f"{sample_slug}-high-large",
|
|
mapping(box(5.005, 51.005, 5.007, 51.007)),
|
|
tile_path=tile_path,
|
|
confidence=0.81,
|
|
bbox={"x_min": 70, "y_min": 18, "x_max": 111, "y_max": 62},
|
|
),
|
|
_feature(
|
|
"match_reference",
|
|
f"{sample_slug}-reference",
|
|
mapping(box(5.002, 51.002, 5.003, 51.003)),
|
|
),
|
|
_feature(
|
|
"false_negative",
|
|
f"{sample_slug}-missed-reference",
|
|
mapping(box(5.006, 51.002, 5.007, 51.003)),
|
|
),
|
|
]
|
|
evidence_dir = tmp_path / "portfolio" / "samples" / sample_slug / "evidence"
|
|
evidence_dir.mkdir(parents=True)
|
|
evidence_path = evidence_dir / "calibration_evidence.geojson"
|
|
evidence_path.write_text(
|
|
json.dumps({"type": "FeatureCollection", "features": features}),
|
|
encoding="utf-8",
|
|
)
|
|
samples.append(
|
|
{
|
|
"sample_slug": sample_slug,
|
|
"aoi_label": sample_slug.title(),
|
|
"role_counts": {
|
|
"false_positive": 3,
|
|
"match_reference": 1,
|
|
"false_negative": 1,
|
|
},
|
|
"evidence_geojson_path": str(evidence_path),
|
|
}
|
|
)
|
|
portfolio_path = tmp_path / "portfolio" / "calibration_evidence_portfolio.json"
|
|
portfolio_path.write_text(
|
|
json.dumps(
|
|
{
|
|
"model_asset_id": "model-review",
|
|
"model_sha256": "abc123",
|
|
"samples": samples,
|
|
}
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
return portfolio_path, storage_root
|
|
|
|
|
|
def test_false_positive_visual_review_is_stratified_and_requires_manual_decisions(
|
|
tmp_path: Path,
|
|
) -> None:
|
|
renderer = ROOT / "scripts" / "render_detection_false_positive_review_contact_sheets.py"
|
|
validator = ROOT / "scripts" / "validate_detection_false_positive_review_decisions.py"
|
|
readiness = (ROOT / "scripts" / "run_readiness_check.sh").read_text(encoding="utf-8")
|
|
dockerfile = (ROOT / "deploy" / "unraid" / "Dockerfile.all-in-one").read_text(
|
|
encoding="utf-8"
|
|
)
|
|
assert renderer.exists()
|
|
assert validator.exists()
|
|
assert "py_compile scripts/render_detection_false_positive_review_contact_sheets.py" in readiness
|
|
assert "py_compile scripts/validate_detection_false_positive_review_decisions.py" in readiness
|
|
assert "COPY scripts/render_detection_false_positive_review_contact_sheets.py" in dockerfile
|
|
assert "COPY scripts/validate_detection_false_positive_review_decisions.py" in dockerfile
|
|
|
|
portfolio_path, storage_root = _write_review_portfolio(tmp_path)
|
|
output_dir = tmp_path / "review"
|
|
result = subprocess.run(
|
|
[
|
|
sys.executable,
|
|
str(renderer),
|
|
"--portfolio",
|
|
str(portfolio_path),
|
|
"--storage-root",
|
|
str(storage_root),
|
|
"--output-dir",
|
|
str(output_dir),
|
|
"--sample-slugs",
|
|
"geel,turnhout",
|
|
"--max-features",
|
|
"4",
|
|
"--columns",
|
|
"2",
|
|
"--cards-per-sheet",
|
|
"4",
|
|
"--thumb-size",
|
|
"128",
|
|
],
|
|
cwd=ROOT,
|
|
check=True,
|
|
capture_output=True,
|
|
text=True,
|
|
)
|
|
|
|
report = json.loads(
|
|
(output_dir / "detection_false_positive_review_summary.json").read_text(
|
|
encoding="utf-8"
|
|
)
|
|
)
|
|
assert report["status"] == "review_required"
|
|
assert report["population_count"] == 6
|
|
assert report["selected_feature_count"] == 4
|
|
assert report["selected_sample_slugs"] == ["geel", "turnhout"]
|
|
assert report["missing_provenance_count"] == 0
|
|
assert report["missing_tile_count"] == 0
|
|
assert report["reference_overlay_feature_count"] > 0
|
|
assert set(report["selected_area_buckets"])
|
|
assert set(report["selected_confidence_bands"])
|
|
assert "review required" in result.stdout.lower()
|
|
|
|
sheet_path = output_dir / report["contact_sheets"][0]["path"]
|
|
sheet = Image.open(sheet_path).convert("RGB")
|
|
assert sheet.width >= 256
|
|
assert sheet.height >= 256
|
|
assert len(sheet.getcolors(maxcolors=1_000_000) or []) > 20
|
|
|
|
decisions_path = output_dir / "false_positive_review_decisions.csv"
|
|
with decisions_path.open(newline="", encoding="utf-8") as handle:
|
|
rows = list(csv.DictReader(handle))
|
|
assert len(rows) == 4
|
|
assert {row["review_decision"] for row in rows} == {"unreviewed"}
|
|
assert all(row["candidate_feature_id"] for row in rows)
|
|
assert all(row["source_tile_path"] for row in rows)
|
|
|
|
incomplete_dir = tmp_path / "incomplete"
|
|
incomplete = subprocess.run(
|
|
[
|
|
sys.executable,
|
|
str(validator),
|
|
"--review-summary",
|
|
str(output_dir / "detection_false_positive_review_summary.json"),
|
|
"--decisions-csv",
|
|
str(decisions_path),
|
|
"--output-dir",
|
|
str(incomplete_dir),
|
|
"--require-complete",
|
|
],
|
|
cwd=ROOT,
|
|
check=False,
|
|
capture_output=True,
|
|
text=True,
|
|
)
|
|
assert incomplete.returncode == 2
|
|
incomplete_validation = json.loads(
|
|
(incomplete_dir / "detection_false_positive_review_validation.json").read_text(
|
|
encoding="utf-8"
|
|
)
|
|
)
|
|
assert incomplete_validation["status"] == "review_required"
|
|
assert incomplete_validation["decision_counts"]["unreviewed"] == 4
|
|
incomplete_confirmed = json.loads(
|
|
(incomplete_dir / "confirmed_model_false_positives.geojson").read_text(
|
|
encoding="utf-8"
|
|
)
|
|
)
|
|
assert incomplete_confirmed["features"] == []
|
|
|
|
decisions = (
|
|
"confirmed_model_false_positive",
|
|
"reference_gap_or_change",
|
|
"qa_alignment_mismatch",
|
|
"uncertain",
|
|
)
|
|
for row, decision in zip(rows, decisions, strict=True):
|
|
row["review_decision"] = decision
|
|
row["review_notes"] = f"reviewed as {decision}"
|
|
with decisions_path.open("w", newline="", encoding="utf-8") as handle:
|
|
writer = csv.DictWriter(handle, fieldnames=list(rows[0]))
|
|
writer.writeheader()
|
|
writer.writerows(rows)
|
|
|
|
validation_dir = tmp_path / "validated"
|
|
subprocess.run(
|
|
[
|
|
sys.executable,
|
|
str(validator),
|
|
"--review-summary",
|
|
str(output_dir / "detection_false_positive_review_summary.json"),
|
|
"--decisions-csv",
|
|
str(decisions_path),
|
|
"--output-dir",
|
|
str(validation_dir),
|
|
"--require-complete",
|
|
],
|
|
cwd=ROOT,
|
|
check=True,
|
|
capture_output=True,
|
|
text=True,
|
|
)
|
|
validation = json.loads(
|
|
(validation_dir / "detection_false_positive_review_validation.json").read_text(
|
|
encoding="utf-8"
|
|
)
|
|
)
|
|
assert validation["status"] == "complete"
|
|
assert validation["decision_counts"] == {
|
|
"confirmed_model_false_positive": 1,
|
|
"qa_alignment_mismatch": 1,
|
|
"reference_gap_or_change": 1,
|
|
"uncertain": 1,
|
|
"unreviewed": 0,
|
|
}
|
|
confirmed = json.loads(
|
|
(validation_dir / "confirmed_model_false_positives.geojson").read_text(
|
|
encoding="utf-8"
|
|
)
|
|
)
|
|
assert len(confirmed["features"]) == 1
|
|
assert confirmed["features"][0]["properties"]["review_decision"] == (
|
|
"confirmed_model_false_positive"
|
|
)
|
|
|
|
|
|
def test_false_positive_visual_review_rejects_tiles_outside_storage_root(
|
|
tmp_path: Path,
|
|
) -> None:
|
|
renderer = ROOT / "scripts" / "render_detection_false_positive_review_contact_sheets.py"
|
|
portfolio_path, storage_root = _write_review_portfolio(tmp_path, unsafe_tile=True)
|
|
|
|
result = subprocess.run(
|
|
[
|
|
sys.executable,
|
|
str(renderer),
|
|
"--portfolio",
|
|
str(portfolio_path),
|
|
"--storage-root",
|
|
str(storage_root),
|
|
"--output-dir",
|
|
str(tmp_path / "review"),
|
|
"--sample-slugs",
|
|
"geel",
|
|
"--max-features",
|
|
"3",
|
|
],
|
|
cwd=ROOT,
|
|
check=False,
|
|
capture_output=True,
|
|
text=True,
|
|
)
|
|
|
|
assert result.returncode != 0
|
|
assert "outside storage root" in result.stderr
|