226 lines
6.9 KiB
Python
226 lines
6.9 KiB
Python
from __future__ import annotations
|
|
|
|
import json
|
|
import subprocess
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
|
|
ROOT = Path(__file__).resolve().parents[2]
|
|
|
|
|
|
def _polygon(x: float, y: float, size: float) -> dict:
|
|
return {
|
|
"type": "Polygon",
|
|
"coordinates": [
|
|
[
|
|
[x, y],
|
|
[x + size, y],
|
|
[x + size, y + size],
|
|
[x, y + size],
|
|
[x, y],
|
|
]
|
|
],
|
|
}
|
|
|
|
|
|
def _evidence_feature(
|
|
role: str,
|
|
feature_id: str,
|
|
geometry: dict,
|
|
*,
|
|
tile_index: int = 1,
|
|
confidence: float | None = None,
|
|
) -> dict:
|
|
properties = {
|
|
"qa_evidence_role": role,
|
|
"candidate_feature_id": feature_id,
|
|
"feature_class": "building",
|
|
"tile_index": tile_index,
|
|
"analysis_run_id": "run-1",
|
|
"quality_check_id": "quality-1",
|
|
"calibration_threshold": 0.15,
|
|
"calibration_model_asset_id": "model-a",
|
|
}
|
|
if confidence is not None:
|
|
properties["candidate_confidence"] = confidence
|
|
return {
|
|
"type": "Feature",
|
|
"id": f"{role}:{feature_id}",
|
|
"properties": properties,
|
|
"geometry": geometry,
|
|
}
|
|
|
|
|
|
def _write_portfolio(
|
|
tmp_path: Path,
|
|
features: list[dict],
|
|
*,
|
|
declared_false_positives: int,
|
|
) -> Path:
|
|
evidence_dir = tmp_path / "samples" / "geel" / "evidence"
|
|
evidence_dir.mkdir(parents=True)
|
|
evidence_path = evidence_dir / "calibration_evidence.geojson"
|
|
evidence_path.write_text(
|
|
json.dumps({"type": "FeatureCollection", "features": features}),
|
|
encoding="utf-8",
|
|
)
|
|
portfolio_path = tmp_path / "calibration_evidence_portfolio.json"
|
|
portfolio_path.write_text(
|
|
json.dumps(
|
|
{
|
|
"model_asset_id": "model-a",
|
|
"model_sha256": "abc123",
|
|
"samples": [
|
|
{
|
|
"sample_slug": "geel",
|
|
"role_counts": {
|
|
"false_positive": declared_false_positives,
|
|
"match_candidate": 1,
|
|
},
|
|
"evidence_geojson_path": str(evidence_path),
|
|
}
|
|
],
|
|
}
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
return portfolio_path
|
|
|
|
|
|
def test_detection_results_table_uses_bounded_local_pagination() -> None:
|
|
lab = (ROOT / "frontend" / "src" / "components" / "detection" / "DetectionLab.tsx").read_text(
|
|
encoding="utf-8"
|
|
)
|
|
styles = (ROOT / "frontend" / "src" / "styles" / "app.css").read_text(
|
|
encoding="utf-8"
|
|
)
|
|
|
|
assert "DETECTION_PAGE_SIZE_OPTIONS" in lab
|
|
assert "visibleDetectionItems" in lab
|
|
assert "detectionItems.slice" in lab
|
|
assert "Detection result pagination" in lab
|
|
assert "Vorige resultatenpagina" in lab
|
|
assert "Volgende resultatenpagina" in lab
|
|
assert "formatSourceTilePath(detection.source_tile_path)" in lab
|
|
assert 'title={detection.source_tile_path ?? undefined}' in lab
|
|
assert "pagination-toolbar" in styles
|
|
assert ".source-tile-cell" in styles
|
|
assert "detectionItems.map((detection)" not in lab
|
|
|
|
|
|
def test_false_positive_audit_builds_reviewable_persisted_evidence(tmp_path: Path) -> None:
|
|
script = ROOT / "scripts" / "audit_detection_false_positive_evidence.py"
|
|
readiness = (ROOT / "scripts" / "run_readiness_check.sh").read_text(encoding="utf-8")
|
|
dockerfile = (ROOT / "deploy" / "unraid" / "Dockerfile.all-in-one").read_text(
|
|
encoding="utf-8"
|
|
)
|
|
assert script.exists()
|
|
assert "py_compile scripts/audit_detection_false_positive_evidence.py" in readiness
|
|
assert "COPY scripts/audit_detection_false_positive_evidence.py" in dockerfile
|
|
|
|
features = [
|
|
_evidence_feature(
|
|
"false_positive",
|
|
"fp-small",
|
|
_polygon(5.0, 51.2, 0.0001),
|
|
tile_index=4,
|
|
confidence=0.27,
|
|
),
|
|
_evidence_feature(
|
|
"false_positive",
|
|
"fp-large",
|
|
_polygon(5.001, 51.2, 0.0003),
|
|
tile_index=4,
|
|
confidence=0.81,
|
|
),
|
|
_evidence_feature(
|
|
"match_candidate",
|
|
"matched",
|
|
_polygon(5.002, 51.2, 0.0002),
|
|
tile_index=7,
|
|
),
|
|
]
|
|
portfolio_path = _write_portfolio(
|
|
tmp_path / "portfolio",
|
|
features,
|
|
declared_false_positives=2,
|
|
)
|
|
output_dir = tmp_path / "audit"
|
|
result = subprocess.run(
|
|
[
|
|
sys.executable,
|
|
str(script),
|
|
"--portfolio",
|
|
str(portfolio_path),
|
|
"--output-dir",
|
|
str(output_dir),
|
|
],
|
|
cwd=ROOT,
|
|
check=True,
|
|
capture_output=True,
|
|
text=True,
|
|
)
|
|
|
|
report = json.loads(
|
|
(output_dir / "detection_false_positive_audit.json").read_text(encoding="utf-8")
|
|
)
|
|
assert report["model_asset_id"] == "model-a"
|
|
assert report["model_sha256"] == "abc123"
|
|
assert report["false_positive_count"] == 2
|
|
assert report["candidate_count"] == 3
|
|
assert report["false_positive_rate"] == 2 / 3
|
|
assert report["confidence_coverage_count"] == 2
|
|
assert report["confidence"]["median"] == 0.54
|
|
assert report["source_tile_counts"] == {"geel:4": 2}
|
|
assert sum(bucket["count"] for bucket in report["area_buckets"].values()) == 2
|
|
assert report["samples"][0]["false_positive_area_m2"]["median"] > 0
|
|
assert report["recommendations"]
|
|
|
|
geojson = json.loads(
|
|
(output_dir / "false_positives.geojson").read_text(encoding="utf-8")
|
|
)
|
|
assert geojson["type"] == "FeatureCollection"
|
|
assert len(geojson["features"]) == 2
|
|
assert all(
|
|
feature["properties"]["qa_evidence_role"] == "false_positive"
|
|
for feature in geojson["features"]
|
|
)
|
|
assert all(feature["properties"]["sample_slug"] == "geel" for feature in geojson["features"])
|
|
assert all(feature["properties"]["area_m2"] > 0 for feature in geojson["features"])
|
|
assert "False-positive audit JSON" in result.stdout
|
|
assert (output_dir / "detection_false_positive_audit.md").is_file()
|
|
|
|
|
|
def test_false_positive_audit_rejects_declared_role_count_drift(tmp_path: Path) -> None:
|
|
script = ROOT / "scripts" / "audit_detection_false_positive_evidence.py"
|
|
portfolio_path = _write_portfolio(
|
|
tmp_path / "portfolio",
|
|
[
|
|
_evidence_feature(
|
|
"false_positive",
|
|
"fp-one",
|
|
_polygon(5.0, 51.2, 0.0001),
|
|
)
|
|
],
|
|
declared_false_positives=2,
|
|
)
|
|
|
|
result = subprocess.run(
|
|
[
|
|
sys.executable,
|
|
str(script),
|
|
"--portfolio",
|
|
str(portfolio_path),
|
|
"--output-dir",
|
|
str(tmp_path / "audit"),
|
|
],
|
|
cwd=ROOT,
|
|
check=False,
|
|
capture_output=True,
|
|
text=True,
|
|
)
|
|
|
|
assert result.returncode != 0
|
|
assert "declares 2 false positives but evidence contains 1" in result.stderr
|