feat(accuracy): build hardened phase 4 evaluation harness
This commit is contained in:
Jens
@@ -0,0 +1,397 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
import hashlib
|
||||
import json
|
||||
import math
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
SCRIPTS = ROOT / "scripts"
|
||||
if str(SCRIPTS) not in sys.path:
|
||||
sys.path.insert(0, str(SCRIPTS))
|
||||
|
||||
from accuracy_phase4_evaluator import ( # noqa: E402
|
||||
TASKS,
|
||||
canonical_hash,
|
||||
count_metrics,
|
||||
detection_ap,
|
||||
evaluate_cases,
|
||||
evaluate_object_detection,
|
||||
evaluate_footprint_segmentation,
|
||||
evaluate_raster_classification,
|
||||
evaluate_terrain,
|
||||
evaluate_validation,
|
||||
evaluate_vector_comparison,
|
||||
subgroup_report,
|
||||
task_inventory,
|
||||
)
|
||||
|
||||
|
||||
METADATA = {
|
||||
"region": "flanders",
|
||||
"municipality": "Mol",
|
||||
"urbanity": "urban",
|
||||
"object_size": "medium",
|
||||
"source": "synthetic-source",
|
||||
"sensor": "synthetic-sensor",
|
||||
"resolution_m": 0.25,
|
||||
"season": "summer",
|
||||
"date": "2026-01-01",
|
||||
"vegetation": "partial",
|
||||
"occlusion": "none",
|
||||
"difficulty": "normal",
|
||||
}
|
||||
|
||||
|
||||
def lineage(sample_id: str) -> dict:
|
||||
return {
|
||||
"reference": {
|
||||
"source_id": f"synthetic:{sample_id}:reference",
|
||||
"source_version": "1",
|
||||
"derivation": "hand_authored_contract_fixture",
|
||||
},
|
||||
"prediction": {
|
||||
"source_id": f"synthetic:{sample_id}:prediction",
|
||||
"source_version": "1",
|
||||
"derivation": "hand_authored_fixed_output",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def detection_case(sample_id: str = "det-1") -> dict:
|
||||
return {
|
||||
"sample_id": sample_id,
|
||||
"task": "object_detection",
|
||||
"split": "test",
|
||||
"metadata": copy.deepcopy(METADATA),
|
||||
"config": {"confidence_threshold": 0.5, "match_iou": 0.5},
|
||||
"lineage": lineage(sample_id),
|
||||
"classes": ["building", "tank"],
|
||||
"references": [{"id": "r-building", "class": "building", "bbox": [0, 0, 4, 4]}],
|
||||
"predictions": [
|
||||
{
|
||||
"id": "p-building",
|
||||
"class": "building",
|
||||
"bbox": [0, 0, 4, 4],
|
||||
"confidence": 0.8,
|
||||
},
|
||||
{
|
||||
"id": "p-filtered",
|
||||
"class": "building",
|
||||
"bbox": [10, 10, 12, 12],
|
||||
"confidence": 0.2,
|
||||
},
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def raster_case(sample_id: str = "raster-1") -> dict:
|
||||
reference_side = {
|
||||
"crs": "EPSG:31370",
|
||||
"transform": [1, 0, 100000, 0, -1, 200000],
|
||||
"shape": [2, 2],
|
||||
"nodata": -9999,
|
||||
"mask": [[True, True], [True, True]],
|
||||
}
|
||||
return {
|
||||
"sample_id": sample_id,
|
||||
"task": "raster_classification",
|
||||
"split": "test",
|
||||
"metadata": copy.deepcopy(METADATA),
|
||||
"config": {},
|
||||
"lineage": lineage(sample_id),
|
||||
"classes": [0, 1],
|
||||
"references": [[0, 1], [1, 0]],
|
||||
"predictions": [[0, 1], [1, 0]],
|
||||
"raster_context": {
|
||||
"reference": reference_side,
|
||||
"prediction": copy.deepcopy(reference_side),
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def polygon_case(task: str = "vector_comparison") -> dict:
|
||||
sample_id = f"{task}-1"
|
||||
config = {"match_iou": 0.5}
|
||||
if task == "footprint_segmentation":
|
||||
config["boundary_tolerance_m"] = 1.0
|
||||
polygon = [
|
||||
[100000, 200000],
|
||||
[100010, 200000],
|
||||
[100010, 200010],
|
||||
[100000, 200010],
|
||||
[100000, 200000],
|
||||
]
|
||||
return {
|
||||
"sample_id": sample_id,
|
||||
"task": task,
|
||||
"split": "test",
|
||||
"metadata": copy.deepcopy(METADATA),
|
||||
"config": config,
|
||||
"lineage": lineage(sample_id),
|
||||
"classes": ["building"],
|
||||
"spatial_context": {
|
||||
"crs": "EPSG:31370",
|
||||
"coordinate_units": "m",
|
||||
"metric": True,
|
||||
},
|
||||
"references": [{"id": "reference", "class": "building", "polygon": polygon}],
|
||||
"predictions": [{"id": "prediction", "class": "building", "polygon": polygon}],
|
||||
}
|
||||
|
||||
|
||||
def test_raw_evidence_and_hashes_are_exact_and_recomputable(tmp_path: Path) -> None:
|
||||
case = detection_case()
|
||||
portfolio = {
|
||||
"schema_version": 2,
|
||||
"portfolio_id": "synthetic-hardening-test",
|
||||
"portfolio_lineage": {
|
||||
"origin": "repository_fixture",
|
||||
"source_path": "synthetic.json",
|
||||
"version": "1",
|
||||
},
|
||||
"split_roles": ["test"],
|
||||
"selection_policy": "Fixed before evaluation; no selection.",
|
||||
"claim_boundary": "Synthetic evaluator test; not product accuracy.",
|
||||
"protected_policy": {"threshold_selection_allowed": False},
|
||||
"cases": [case],
|
||||
}
|
||||
path = tmp_path / "portfolio.json"
|
||||
path.write_text(
|
||||
json.dumps(portfolio, ensure_ascii=False, indent=2) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
report = evaluate_cases(path, {case["sample_id"]})
|
||||
raw = report["results"][0]["raw"]
|
||||
|
||||
assert raw["references"] == case["references"]
|
||||
assert raw["predictions_pre_filter"] == case["predictions"]
|
||||
assert raw["predictions_post_filter"] == case["predictions"][:1]
|
||||
assert raw["config"] == case["config"]
|
||||
assert raw["split"] == "test"
|
||||
assert raw["input_lineage"] == case["lineage"]
|
||||
assert raw["portfolio_lineage"]["declared"] == portfolio["portfolio_lineage"]
|
||||
assert raw["hashes"]["case_input_canonical_json_sha256"] == canonical_hash(case)
|
||||
assert raw["hashes"]["references_canonical_json_sha256"] == canonical_hash(
|
||||
case["references"]
|
||||
)
|
||||
assert (
|
||||
report["portfolio_file_sha256"] == hashlib.sha256(path.read_bytes()).hexdigest()
|
||||
)
|
||||
assert report["portfolio_canonical_json_sha256"] == canonical_hash(portfolio)
|
||||
assert report["results_canonical_json_sha256"] == canonical_hash(report["results"])
|
||||
high_threshold = next(
|
||||
row
|
||||
for row in report["results"][0]["metrics"]["coverage_risk"]
|
||||
if row["threshold"] == 0.9
|
||||
)
|
||||
assert high_threshold["retained_prediction_coverage"] == 0.0
|
||||
assert high_threshold["reference_coverage"] == 0.0
|
||||
assert high_threshold["false_negative_count"] == 1
|
||||
assert high_threshold["risk"] == 1.0
|
||||
|
||||
challenge_exposed = copy.deepcopy(portfolio)
|
||||
challenge_exposed["challenge_labels"] = []
|
||||
path.write_text(json.dumps(challenge_exposed, ensure_ascii=False), encoding="utf-8")
|
||||
with pytest.raises(ValueError, match="Challenge cases and labels"):
|
||||
evaluate_cases(path, {case["sample_id"]})
|
||||
|
||||
|
||||
def test_ap_ties_use_stable_ids_and_matching_is_class_aware() -> None:
|
||||
references = [{"id": "r", "class": "building", "bbox": [0, 0, 4, 4]}]
|
||||
predictions = [
|
||||
{
|
||||
"id": "z-true",
|
||||
"class": "building",
|
||||
"bbox": [0, 0, 4, 4],
|
||||
"confidence": 0.8,
|
||||
},
|
||||
{
|
||||
"id": "a-false",
|
||||
"class": "building",
|
||||
"bbox": [10, 10, 12, 12],
|
||||
"confidence": 0.8,
|
||||
},
|
||||
]
|
||||
forward = detection_ap(predictions, references, 0.5)
|
||||
reverse = detection_ap(list(reversed(predictions)), references, 0.5)
|
||||
assert forward == reverse == pytest.approx(0.5)
|
||||
|
||||
wrong_class = copy.deepcopy(predictions)
|
||||
wrong_class[1] = {
|
||||
"id": "a-tank",
|
||||
"class": "tank",
|
||||
"bbox": [0, 0, 4, 4],
|
||||
"confidence": 0.95,
|
||||
}
|
||||
assert detection_ap(wrong_class, references, 0.5) == pytest.approx(0.5)
|
||||
|
||||
|
||||
def test_raster_requires_exact_rectangular_alignment_masks_nodata_and_classes() -> None:
|
||||
invalid_case = detection_case("invalid-class")
|
||||
invalid_case["predictions"][0]["class"] = "road"
|
||||
with pytest.raises(ValueError, match="outside the declared ontology"):
|
||||
evaluate_object_detection(invalid_case)
|
||||
|
||||
valid = raster_case()
|
||||
valid["predictions"][0][1] = -9999
|
||||
valid["raster_context"]["prediction"]["mask"][0][1] = False
|
||||
result = evaluate_raster_classification(valid)
|
||||
assert result["metrics"]["prediction_coverage"] == pytest.approx(0.75)
|
||||
assert result["metrics"]["per_class"]["1"]["false_negative"] == 1
|
||||
|
||||
jagged = raster_case("jagged")
|
||||
jagged["predictions"][1].pop()
|
||||
with pytest.raises(ValueError, match="exactly rectangular"):
|
||||
evaluate_raster_classification(jagged)
|
||||
|
||||
missing_metadata = raster_case("missing-metadata")
|
||||
del missing_metadata["raster_context"]["prediction"]["crs"]
|
||||
with pytest.raises(ValueError, match="missing"):
|
||||
evaluate_raster_classification(missing_metadata)
|
||||
|
||||
shifted = raster_case("shifted")
|
||||
shifted["raster_context"]["prediction"]["transform"][2] += 1
|
||||
with pytest.raises(ValueError, match="affine alignment differs"):
|
||||
evaluate_raster_classification(shifted)
|
||||
|
||||
invalid_class = raster_case("invalid-class")
|
||||
invalid_class["predictions"][0][0] = 3
|
||||
with pytest.raises(ValueError, match="prediction class outside ontology"):
|
||||
evaluate_raster_classification(invalid_class)
|
||||
|
||||
invalid_nodata = raster_case("invalid-nodata")
|
||||
invalid_nodata["predictions"][0][0] = -9999
|
||||
with pytest.raises(ValueError, match="marks nodata as valid"):
|
||||
evaluate_raster_classification(invalid_nodata)
|
||||
|
||||
|
||||
def test_polygon_metrics_require_valid_geometry_projected_crs_and_metres() -> None:
|
||||
assert evaluate_vector_comparison(polygon_case())["metrics"]["f1"] == 1.0
|
||||
assert (
|
||||
evaluate_footprint_segmentation(polygon_case("footprint_segmentation"))[
|
||||
"metrics"
|
||||
]["mean_iou"]
|
||||
== 1.0
|
||||
)
|
||||
|
||||
geographic = polygon_case()
|
||||
geographic["spatial_context"]["crs"] = "EPSG:4326"
|
||||
with pytest.raises(ValueError, match="projected CRS"):
|
||||
evaluate_vector_comparison(geographic)
|
||||
|
||||
wrong_units = polygon_case()
|
||||
wrong_units["spatial_context"]["coordinate_units"] = "degree"
|
||||
with pytest.raises(ValueError, match="must be 'm'"):
|
||||
evaluate_vector_comparison(wrong_units)
|
||||
|
||||
bowtie = polygon_case()
|
||||
bowtie["predictions"][0]["polygon"] = [
|
||||
[100000, 200000],
|
||||
[100010, 200010],
|
||||
[100010, 200000],
|
||||
[100000, 200010],
|
||||
[100000, 200000],
|
||||
]
|
||||
with pytest.raises(ValueError, match="positive-area and valid"):
|
||||
evaluate_vector_comparison(bowtie)
|
||||
|
||||
|
||||
def test_terrain_rejects_non_finite_and_validation_counts_only_critical_misses() -> (
|
||||
None
|
||||
):
|
||||
terrain = {
|
||||
"sample_id": "terrain",
|
||||
"task": "terrain_interpretation",
|
||||
"split": "test",
|
||||
"metadata": copy.deepcopy(METADATA),
|
||||
"config": {},
|
||||
"lineage": lineage("terrain"),
|
||||
"units": "m_TAW",
|
||||
"references": [1.0, 2.0],
|
||||
"predictions": [1.1, None],
|
||||
}
|
||||
assert evaluate_terrain(terrain)["metrics"]["coverage"] == 0.5
|
||||
for field, value in (("references", math.nan), ("predictions", math.inf)):
|
||||
invalid = copy.deepcopy(terrain)
|
||||
invalid[field][0] = value
|
||||
with pytest.raises(ValueError, match="finite number"):
|
||||
evaluate_terrain(invalid)
|
||||
|
||||
validation = {
|
||||
"sample_id": "validation",
|
||||
"task": "geospatial_data_validation",
|
||||
"split": "test",
|
||||
"metadata": copy.deepcopy(METADATA),
|
||||
"config": {},
|
||||
"lineage": lineage("validation"),
|
||||
"expected_anomalies": [{"code": "D-MAJOR", "severity": "major"}],
|
||||
"observed_anomalies": [],
|
||||
}
|
||||
assert (
|
||||
evaluate_validation(validation)["metrics"]["blocker_or_critical_miss_count"]
|
||||
== 0
|
||||
)
|
||||
validation["expected_anomalies"].append(
|
||||
{"code": "D-CRITICAL", "severity": "critical"}
|
||||
)
|
||||
assert (
|
||||
evaluate_validation(validation)["metrics"]["blocker_or_critical_miss_count"]
|
||||
== 1
|
||||
)
|
||||
validation["expected_anomalies"] = ["D-NO-SEVERITY"]
|
||||
with pytest.raises(ValueError, match="include code and severity"):
|
||||
evaluate_validation(validation)
|
||||
|
||||
|
||||
def _subgroup_result(region: str, tp: int, fp: int, fn: int) -> dict:
|
||||
metadata = copy.deepcopy(METADATA)
|
||||
metadata["region"] = region
|
||||
return {
|
||||
"task": "object_detection",
|
||||
"metadata": metadata,
|
||||
"metrics": {**count_metrics(tp, fp, fn), "ap50": 0.5, "ap50_95": 0.4},
|
||||
"failures": [],
|
||||
}
|
||||
|
||||
|
||||
def test_subgroups_report_task_metrics_support_ci_and_worst_stratum() -> None:
|
||||
results = [
|
||||
*[_subgroup_result("strong", 10, 0, 0) for _ in range(5)],
|
||||
*[_subgroup_result("weak", 1, 4, 4) for _ in range(5)],
|
||||
]
|
||||
report = subgroup_report(results)
|
||||
region = report["dimensions"]["region"]
|
||||
weak = region["strata"]["weak"]["task_metrics"]["object_detection"]
|
||||
|
||||
assert weak["status"] == "evaluable"
|
||||
assert weak["case_support"] == 5
|
||||
assert weak["micro"]["precision_ci95_wilson"]["status"] == "computed"
|
||||
assert weak["macro"]["f1_case_support"] == 5
|
||||
assert region["worst_stratum_by_task"]["object_detection"]["stratum"] == "weak"
|
||||
|
||||
insufficient = subgroup_report([_subgroup_result("thin", 1, 0, 0)])
|
||||
thin = insufficient["dimensions"]["region"]["strata"]["thin"]
|
||||
assert thin["task_metrics"]["object_detection"]["status"] == "insufficient_support"
|
||||
assert thin["release_gate_status"] == "not_evaluable"
|
||||
assert insufficient["overall_status"] == "not_evaluable"
|
||||
|
||||
|
||||
def test_capability_inventory_is_comprehensive_and_honest() -> None:
|
||||
inventory = task_inventory()
|
||||
assert {item["task"] for item in inventory} == TASKS
|
||||
assert len(inventory) >= 15
|
||||
assert all(item["implementation_paths"] for item in inventory)
|
||||
assert all(item["suitable_metrics"] for item in inventory)
|
||||
assistant = next(
|
||||
item
|
||||
for item in inventory
|
||||
if item["capability_id"] == "geo_assistant_orchestration"
|
||||
)
|
||||
assert assistant["evaluation_status"].startswith("no_independent_accuracy_score")
|
||||
Reference in New Issue
Block a user