267 lines
8.4 KiB
Python
267 lines
8.4 KiB
Python
from __future__ import annotations
|
|
|
|
import csv
|
|
import json
|
|
import subprocess
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
import numpy as np
|
|
import rasterio
|
|
from PIL import Image
|
|
from rasterio.transform import from_bounds
|
|
from shapely.geometry import box, mapping
|
|
|
|
|
|
ROOT = Path(__file__).resolve().parents[2]
|
|
|
|
|
|
def _write_tile(path: Path) -> None:
|
|
data = np.full((3, 256, 256), 72, dtype=np.uint8)
|
|
data[:, 45:105, 40:110] = np.array([188, 178, 163], dtype=np.uint8)[:, None, None]
|
|
data[:, 145:205, 150:225] = np.array([205, 198, 184], dtype=np.uint8)[:, None, None]
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
with rasterio.open(
|
|
path,
|
|
"w",
|
|
driver="GTiff",
|
|
width=256,
|
|
height=256,
|
|
count=3,
|
|
dtype="uint8",
|
|
crs="EPSG:4326",
|
|
transform=from_bounds(5.0, 51.0, 5.01, 51.01, 256, 256),
|
|
) as dataset:
|
|
dataset.write(data)
|
|
|
|
|
|
def _feature(role: str, feature_id: str, geometry: dict) -> dict:
|
|
return {
|
|
"type": "Feature",
|
|
"id": f"{role}:{feature_id}",
|
|
"properties": {
|
|
"qa_evidence_role": role,
|
|
"feature_id": feature_id,
|
|
"reference_feature_id": feature_id
|
|
if role in {"false_negative", "match_reference"}
|
|
else None,
|
|
"candidate_feature_id": feature_id
|
|
if role in {"false_positive", "match_candidate"}
|
|
else None,
|
|
"analysis_run_id": "run-mol-review",
|
|
"quality_check_id": "quality-mol-review",
|
|
},
|
|
"geometry": geometry,
|
|
}
|
|
|
|
|
|
def _write_portfolio(tmp_path: Path, *, escaped_manifest: bool = False) -> tuple[Path, Path]:
|
|
storage_root = tmp_path / "storage"
|
|
tile_path = storage_root / "tiles" / "mol_donk" / "tile_0000.tif"
|
|
_write_tile(tile_path)
|
|
manifest_path = storage_root / "tiles" / "mol_donk" / "manifest.json"
|
|
if escaped_manifest:
|
|
manifest_path = tmp_path / "outside-manifest.json"
|
|
manifest_path.write_text(
|
|
json.dumps(
|
|
{
|
|
"tiles": [
|
|
{
|
|
"path": str(tile_path),
|
|
"bounds": [5.0, 51.0, 5.01, 51.01],
|
|
"crs": "EPSG:4326",
|
|
}
|
|
]
|
|
}
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
|
|
portfolio_dir = storage_root / "operator-evidence" / "review" / "portfolio"
|
|
summary_path = portfolio_dir / "samples" / "mol_donk" / "quality_matrix_summary.json"
|
|
summary_path.parent.mkdir(parents=True, exist_ok=True)
|
|
summary_path.write_text(
|
|
json.dumps({"items": [{"manifest_path": str(manifest_path)}]}),
|
|
encoding="utf-8",
|
|
)
|
|
evidence_dir = summary_path.parent / "evidence"
|
|
evidence_dir.mkdir(parents=True)
|
|
false_negatives = [
|
|
_feature(
|
|
"false_negative",
|
|
"miss-tiny",
|
|
mapping(box(5.001, 51.001, 5.00103, 51.00103)),
|
|
),
|
|
_feature(
|
|
"false_negative",
|
|
"miss-small",
|
|
mapping(box(5.002, 51.002, 5.0021, 51.0021)),
|
|
),
|
|
_feature(
|
|
"false_negative",
|
|
"miss-medium",
|
|
mapping(box(5.004, 51.004, 5.00418, 51.00418)),
|
|
),
|
|
_feature(
|
|
"false_negative",
|
|
"miss-large",
|
|
mapping(box(5.006, 51.006, 5.007, 51.007)),
|
|
),
|
|
_feature(
|
|
"false_negative",
|
|
"outside-source-raster",
|
|
mapping(box(5.02, 51.02, 5.021, 51.021)),
|
|
),
|
|
]
|
|
evidence_path = evidence_dir / "calibration_evidence.geojson"
|
|
evidence_path.write_text(
|
|
json.dumps(
|
|
{
|
|
"type": "FeatureCollection",
|
|
"features": false_negatives
|
|
+ [
|
|
_feature(
|
|
"match_candidate",
|
|
"candidate-nearby",
|
|
mapping(box(5.003, 51.003, 5.004, 51.004)),
|
|
),
|
|
_feature(
|
|
"match_reference",
|
|
"reference-nearby",
|
|
mapping(box(5.0031, 51.0031, 5.0041, 51.0041)),
|
|
),
|
|
],
|
|
}
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
portfolio_path = portfolio_dir / "calibration_evidence_portfolio.json"
|
|
portfolio_path.write_text(
|
|
json.dumps(
|
|
{
|
|
"model_asset_id": "model-mol-review",
|
|
"samples": [
|
|
{
|
|
"sample_slug": "mol_donk",
|
|
"copied_summary_path": str(summary_path),
|
|
"evidence_geojson_path": str(evidence_path),
|
|
"role_counts": {"false_negative": 5},
|
|
}
|
|
],
|
|
}
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
return portfolio_path, storage_root
|
|
|
|
|
|
def test_false_negative_visual_review_uses_persisted_manifest_and_requires_decisions(
|
|
tmp_path: Path,
|
|
) -> None:
|
|
renderer = ROOT / "scripts" / "render_detection_false_negative_review_contact_sheets.py"
|
|
readiness = (ROOT / "scripts" / "run_readiness_check.sh").read_text(encoding="utf-8")
|
|
dockerfile = (ROOT / "deploy" / "unraid" / "Dockerfile.all-in-one").read_text(
|
|
encoding="utf-8"
|
|
)
|
|
assert renderer.exists()
|
|
assert f"py_compile scripts/{renderer.name}" in readiness
|
|
assert f"COPY scripts/{renderer.name}" in dockerfile
|
|
|
|
portfolio_path, storage_root = _write_portfolio(tmp_path)
|
|
output_dir = tmp_path / "review"
|
|
result = subprocess.run(
|
|
[
|
|
sys.executable,
|
|
str(renderer),
|
|
"--portfolio",
|
|
str(portfolio_path),
|
|
"--storage-root",
|
|
str(storage_root),
|
|
"--output-dir",
|
|
str(output_dir),
|
|
"--sample-slugs",
|
|
"mol_donk",
|
|
"--max-features",
|
|
"4",
|
|
"--columns",
|
|
"2",
|
|
"--cards-per-sheet",
|
|
"4",
|
|
"--thumb-size",
|
|
"128",
|
|
],
|
|
cwd=ROOT,
|
|
check=True,
|
|
capture_output=True,
|
|
text=True,
|
|
)
|
|
|
|
report = json.loads(
|
|
(output_dir / "detection_false_negative_review_summary.json").read_text(
|
|
encoding="utf-8"
|
|
)
|
|
)
|
|
assert report["status"] == "review_required"
|
|
assert report["population_count"] == 4
|
|
assert report["evidence_population_count"] == 5
|
|
assert report["selected_feature_count"] == 4
|
|
assert report["selected_sample_slugs"] == ["mol_donk"]
|
|
assert report["missing_manifest_count"] == 0
|
|
assert report["missing_tile_count"] == 0
|
|
assert report["outside_tile_coverage_count"] == 1
|
|
assert report["context_overlay_feature_count"] > 0
|
|
assert len(report["selected_area_buckets"]) >= 3
|
|
assert {Path(item["source_tile_path"]).name for item in report["selected_features"]} == {
|
|
"tile_0000.tif"
|
|
}
|
|
assert "review required" in result.stdout.lower()
|
|
|
|
outside = json.loads(
|
|
(output_dir / "false_negatives_outside_tile_coverage.geojson").read_text(
|
|
encoding="utf-8"
|
|
)
|
|
)
|
|
assert len(outside["features"]) == 1
|
|
assert (
|
|
outside["features"][0]["properties"]["review_exclusion_reason"]
|
|
== "outside_tile_coverage"
|
|
)
|
|
|
|
with (output_dir / "false_negative_review_decisions.csv").open(
|
|
newline="", encoding="utf-8"
|
|
) as handle:
|
|
decisions = list(csv.DictReader(handle))
|
|
assert len(decisions) == 4
|
|
assert {row["review_decision"] for row in decisions} == {"unreviewed"}
|
|
|
|
sheet = Image.open(output_dir / report["contact_sheets"][0]["path"]).convert("RGB")
|
|
assert sheet.width >= 256
|
|
assert sheet.height >= 256
|
|
assert len(sheet.getcolors(maxcolors=1_000_000) or []) > 10
|
|
|
|
|
|
def test_false_negative_visual_review_rejects_manifest_outside_storage(
|
|
tmp_path: Path,
|
|
) -> None:
|
|
renderer = ROOT / "scripts" / "render_detection_false_negative_review_contact_sheets.py"
|
|
portfolio_path, storage_root = _write_portfolio(tmp_path, escaped_manifest=True)
|
|
result = subprocess.run(
|
|
[
|
|
sys.executable,
|
|
str(renderer),
|
|
"--portfolio",
|
|
str(portfolio_path),
|
|
"--storage-root",
|
|
str(storage_root),
|
|
"--output-dir",
|
|
str(tmp_path / "review"),
|
|
],
|
|
cwd=ROOT,
|
|
check=False,
|
|
capture_output=True,
|
|
text=True,
|
|
)
|
|
|
|
assert result.returncode != 0
|
|
assert "Tile manifest is outside storage root" in result.stderr
|