719 lines
25 KiB
Python
719 lines
25 KiB
Python
from __future__ import annotations
|
|
|
|
import copy
|
|
import hashlib
|
|
import json
|
|
import math
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
|
|
ROOT = Path(__file__).resolve().parents[2]
|
|
SCRIPTS = ROOT / "scripts"
|
|
if str(SCRIPTS) not in sys.path:
|
|
sys.path.insert(0, str(SCRIPTS))
|
|
|
|
from accuracy_phase4_evaluator import ( # noqa: E402
|
|
TASKS,
|
|
EXPECTED_PROTECTED_POLICY,
|
|
canonical_hash,
|
|
count_metrics,
|
|
detection_ap,
|
|
evaluate_cases,
|
|
evaluate_object_detection,
|
|
evaluate_footprint_segmentation,
|
|
evaluate_raster_classification,
|
|
evaluate_terrain,
|
|
evaluate_validation,
|
|
evaluate_vector_comparison,
|
|
subgroup_report,
|
|
task_inventory,
|
|
)
|
|
|
|
|
|
METADATA = {
|
|
"region": "flanders",
|
|
"municipality": "Mol",
|
|
"urbanity": "urban",
|
|
"object_size": "medium",
|
|
"source": "synthetic-source",
|
|
"sensor": "synthetic-sensor",
|
|
"resolution_m": 0.25,
|
|
"context": "dense_urban",
|
|
"season": "summer",
|
|
"date": "2026-01-01",
|
|
"vegetation": "partial",
|
|
"occlusion": "none",
|
|
"difficulty": "normal",
|
|
}
|
|
|
|
|
|
def lineage(sample_id: str) -> dict:
|
|
return {
|
|
"reference": {
|
|
"source_id": f"synthetic:{sample_id}:reference",
|
|
"source_version": "1",
|
|
"derivation": "hand_authored_contract_fixture",
|
|
},
|
|
"prediction": {
|
|
"source_id": f"synthetic:{sample_id}:prediction",
|
|
"source_version": "1",
|
|
"derivation": "hand_authored_fixed_output",
|
|
},
|
|
}
|
|
|
|
|
|
def detection_case(sample_id: str = "det-1") -> dict:
|
|
return {
|
|
"sample_id": sample_id,
|
|
"task": "object_detection",
|
|
"split": "test",
|
|
"metadata": copy.deepcopy(METADATA),
|
|
"config": {"confidence_threshold": 0.5, "match_iou": 0.5},
|
|
"lineage": lineage(sample_id),
|
|
"classes": ["building", "tank"],
|
|
"references": [{"id": "r-building", "class": "building", "bbox": [0, 0, 4, 4]}],
|
|
"predictions": [
|
|
{
|
|
"id": "p-building",
|
|
"class": "building",
|
|
"bbox": [0, 0, 4, 4],
|
|
"confidence": 0.8,
|
|
},
|
|
{
|
|
"id": "p-filtered",
|
|
"class": "building",
|
|
"bbox": [10, 10, 12, 12],
|
|
"confidence": 0.2,
|
|
},
|
|
],
|
|
}
|
|
|
|
|
|
def raster_case(sample_id: str = "raster-1") -> dict:
|
|
reference_side = {
|
|
"crs": "EPSG:31370",
|
|
"transform": [1, 0, 100000, 0, -1, 200000],
|
|
"shape": [2, 2],
|
|
"nodata": -9999,
|
|
"mask": [[True, True], [True, True]],
|
|
}
|
|
return {
|
|
"sample_id": sample_id,
|
|
"task": "raster_classification",
|
|
"split": "test",
|
|
"metadata": copy.deepcopy(METADATA),
|
|
"config": {},
|
|
"lineage": lineage(sample_id),
|
|
"classes": [0, 1],
|
|
"references": [[0, 1], [1, 0]],
|
|
"predictions": [[0, 1], [1, 0]],
|
|
"raster_context": {
|
|
"reference": reference_side,
|
|
"prediction": copy.deepcopy(reference_side),
|
|
},
|
|
}
|
|
|
|
|
|
def polygon_case(task: str = "vector_comparison") -> dict:
|
|
sample_id = f"{task}-1"
|
|
config = {"match_iou": 0.5}
|
|
if task == "footprint_segmentation":
|
|
config["boundary_tolerance_m"] = 1.0
|
|
polygon = [
|
|
[100000, 200000],
|
|
[100010, 200000],
|
|
[100010, 200010],
|
|
[100000, 200010],
|
|
[100000, 200000],
|
|
]
|
|
return {
|
|
"sample_id": sample_id,
|
|
"task": task,
|
|
"split": "test",
|
|
"metadata": copy.deepcopy(METADATA),
|
|
"config": config,
|
|
"lineage": lineage(sample_id),
|
|
"classes": ["building"],
|
|
"spatial_context": {
|
|
"crs": "EPSG:31370",
|
|
"coordinate_units": "m",
|
|
"metric": True,
|
|
},
|
|
"references": [{"id": "reference", "class": "building", "polygon": polygon}],
|
|
"predictions": [{"id": "prediction", "class": "building", "polygon": polygon}],
|
|
}
|
|
|
|
|
|
def test_raw_evidence_and_hashes_are_exact_and_recomputable(tmp_path: Path) -> None:
|
|
case = detection_case()
|
|
portfolio = {
|
|
"schema_version": 2,
|
|
"portfolio_kind": "synthetic_contract",
|
|
"portfolio_id": "synthetic-hardening-test",
|
|
"portfolio_lineage": {
|
|
"origin": "repository_fixture",
|
|
"source_path": "synthetic.json",
|
|
"version": "1",
|
|
},
|
|
"split_roles": ["test"],
|
|
"selection_policy": "Fixed before evaluation; no selection.",
|
|
"claim_boundary": "Synthetic evaluator test; not product accuracy.",
|
|
"protected_policy": copy.deepcopy(EXPECTED_PROTECTED_POLICY),
|
|
"cases": [case],
|
|
}
|
|
path = tmp_path / "portfolio.json"
|
|
path.write_text(
|
|
json.dumps(portfolio, ensure_ascii=False, indent=2) + "\n",
|
|
encoding="utf-8",
|
|
)
|
|
|
|
report = evaluate_cases(path, {case["sample_id"]})
|
|
raw = report["results"][0]["raw"]
|
|
|
|
assert raw["references"] == case["references"]
|
|
assert raw["predictions_pre_filter"] == case["predictions"]
|
|
assert raw["predictions_post_filter"] == case["predictions"][:1]
|
|
assert raw["config"] == case["config"]
|
|
assert raw["split"] == "test"
|
|
assert raw["input_lineage"] == case["lineage"]
|
|
assert raw["portfolio_lineage"]["declared"] == portfolio["portfolio_lineage"]
|
|
assert raw["hashes"]["case_input_canonical_json_sha256"] == canonical_hash(case)
|
|
assert raw["hashes"]["references_canonical_json_sha256"] == canonical_hash(
|
|
case["references"]
|
|
)
|
|
assert (
|
|
report["portfolio_file_sha256"] == hashlib.sha256(path.read_bytes()).hexdigest()
|
|
)
|
|
assert report["portfolio_canonical_json_sha256"] == canonical_hash(portfolio)
|
|
assert report["results_canonical_json_sha256"] == canonical_hash(report["results"])
|
|
high_threshold = next(
|
|
row
|
|
for row in report["results"][0]["metrics"]["coverage_risk"]
|
|
if row["threshold"] == 0.9
|
|
)
|
|
assert high_threshold["retained_prediction_coverage"] == 0.0
|
|
assert high_threshold["reference_coverage"] == 0.0
|
|
assert high_threshold["false_negative_count"] == 1
|
|
assert high_threshold["risk"] == 1.0
|
|
|
|
challenge_exposed = copy.deepcopy(portfolio)
|
|
challenge_exposed["challenge_labels"] = []
|
|
path.write_text(json.dumps(challenge_exposed, ensure_ascii=False), encoding="utf-8")
|
|
with pytest.raises(ValueError, match="Challenge cases and labels"):
|
|
evaluate_cases(path, {case["sample_id"]})
|
|
|
|
|
|
def test_portfolio_schema_policy_metadata_and_lineage_are_strict(
|
|
tmp_path: Path,
|
|
) -> None:
|
|
case = detection_case("strict-contract")
|
|
portfolio = {
|
|
"schema_version": 2,
|
|
"portfolio_kind": "synthetic_contract",
|
|
"portfolio_id": "synthetic-strict-contract",
|
|
"portfolio_lineage": {
|
|
"origin": "repository_fixture",
|
|
"source_path": "synthetic.json",
|
|
"version": "1",
|
|
},
|
|
"split_roles": ["test"],
|
|
"selection_policy": "Fixed before evaluation; no selection.",
|
|
"claim_boundary": "Synthetic evaluator test; not product accuracy.",
|
|
"protected_policy": copy.deepcopy(EXPECTED_PROTECTED_POLICY),
|
|
"cases": [case],
|
|
}
|
|
path = tmp_path / "strict.json"
|
|
|
|
def evaluate(value: dict) -> dict:
|
|
path.write_text(json.dumps(value, ensure_ascii=False), encoding="utf-8")
|
|
return evaluate_cases(path, {case["sample_id"]})
|
|
|
|
assert evaluate(portfolio)["case_count"] == 1
|
|
|
|
for invalid_version in (1, True, "2"):
|
|
invalid = copy.deepcopy(portfolio)
|
|
invalid["schema_version"] = invalid_version
|
|
with pytest.raises(
|
|
ValueError, match="schema_version must be exactly integer 2"
|
|
):
|
|
evaluate(invalid)
|
|
|
|
invalid_policy = copy.deepcopy(portfolio)
|
|
invalid_policy["protected_policy"]["test_feedback_allowed"] = True
|
|
with pytest.raises(ValueError, match="protected_policy must exactly equal"):
|
|
evaluate(invalid_policy)
|
|
|
|
invalid_metadata = copy.deepcopy(portfolio)
|
|
invalid_metadata["cases"][0]["metadata"]["source"] = "unknown"
|
|
with pytest.raises(ValueError, match="metadata.source must be a meaningful"):
|
|
evaluate(invalid_metadata)
|
|
|
|
invalid_resolution = copy.deepcopy(portfolio)
|
|
invalid_resolution["cases"][0]["metadata"]["resolution_m"] = 0
|
|
with pytest.raises(ValueError, match="metadata.resolution_m must be positive"):
|
|
evaluate(invalid_resolution)
|
|
|
|
invalid_lineage = copy.deepcopy(portfolio)
|
|
del invalid_lineage["cases"][0]["lineage"]["prediction"]["derivation"]
|
|
with pytest.raises(ValueError, match="lineage.prediction missing"):
|
|
evaluate(invalid_lineage)
|
|
|
|
|
|
def test_portfolio_kind_separates_synthetic_and_governed_product_claims(
|
|
tmp_path: Path,
|
|
) -> None:
|
|
fixture_path = (
|
|
ROOT / "fixtures" / "accuracy" / "p4" / "protected-baseline-cases.json"
|
|
)
|
|
synthetic = json.loads(fixture_path.read_text(encoding="utf-8"))
|
|
allowed = {item["sample_id"] for item in synthetic["cases"]}
|
|
path = tmp_path / "portfolio.json"
|
|
|
|
missing_kind = copy.deepcopy(synthetic)
|
|
del missing_kind["portfolio_kind"]
|
|
path.write_text(json.dumps(missing_kind), encoding="utf-8")
|
|
with pytest.raises(ValueError, match="portfolio_kind"):
|
|
evaluate_cases(path, allowed)
|
|
|
|
confused = copy.deepcopy(synthetic)
|
|
confused["claim_boundary"] = "Governed product baseline accuracy evidence."
|
|
path.write_text(json.dumps(confused), encoding="utf-8")
|
|
with pytest.raises(ValueError, match="synthetic_contract"):
|
|
evaluate_cases(path, allowed)
|
|
|
|
governed = json.loads(
|
|
json.dumps(synthetic)
|
|
.replace("Synthetic", "Governed")
|
|
.replace("synthetic", "governed")
|
|
.replace("repository_fixture", "governed_product_evaluation")
|
|
)
|
|
governed["portfolio_kind"] = "governed_product_baseline"
|
|
governed["portfolio_id"] = "governed-product-baseline-test"
|
|
governed["claim_boundary"] = (
|
|
"Governed product baseline metrics recomputed from protected raw cases; "
|
|
"inference provenance is validated separately."
|
|
)
|
|
path.write_text(json.dumps(governed), encoding="utf-8")
|
|
report = evaluate_cases(path, allowed)
|
|
assert report["portfolio_kind"] == "governed_product_baseline"
|
|
assert set(report["evaluated_task_families"]) == TASKS
|
|
|
|
governed["cases"] = governed["cases"][:-1]
|
|
path.write_text(json.dumps(governed), encoding="utf-8")
|
|
|
|
|
|
def test_ap_ties_use_stable_ids_and_matching_is_class_aware() -> None:
|
|
references = [{"id": "r", "class": "building", "bbox": [0, 0, 4, 4]}]
|
|
predictions = [
|
|
{
|
|
"id": "z-true",
|
|
"class": "building",
|
|
"bbox": [0, 0, 4, 4],
|
|
"confidence": 0.8,
|
|
},
|
|
{
|
|
"id": "a-false",
|
|
"class": "building",
|
|
"bbox": [10, 10, 12, 12],
|
|
"confidence": 0.8,
|
|
},
|
|
]
|
|
forward = detection_ap(predictions, references, 0.5)
|
|
reverse = detection_ap(list(reversed(predictions)), references, 0.5)
|
|
assert forward == reverse == pytest.approx(0.5)
|
|
|
|
wrong_class = copy.deepcopy(predictions)
|
|
wrong_class[1] = {
|
|
"id": "a-tank",
|
|
"class": "tank",
|
|
"bbox": [0, 0, 4, 4],
|
|
"confidence": 0.95,
|
|
}
|
|
assert detection_ap(wrong_class, references, 0.5) == pytest.approx(0.5)
|
|
|
|
|
|
def test_detection_ap_and_calibration_are_pooled_globally_and_per_subgroup(
|
|
tmp_path: Path,
|
|
) -> None:
|
|
first = detection_case("a-case")
|
|
first["predictions"] = [
|
|
{
|
|
"id": "p-true",
|
|
"class": "building",
|
|
"bbox": [0, 0, 4, 4],
|
|
"confidence": 0.9,
|
|
}
|
|
]
|
|
second = detection_case("b-case")
|
|
second["predictions"] = [
|
|
{
|
|
"id": "p-false",
|
|
"class": "building",
|
|
"bbox": [10, 10, 12, 12],
|
|
"confidence": 0.9,
|
|
},
|
|
{
|
|
"id": "p-true",
|
|
"class": "building",
|
|
"bbox": [0, 0, 4, 4],
|
|
"confidence": 0.8,
|
|
},
|
|
]
|
|
portfolio = {
|
|
"schema_version": 2,
|
|
"portfolio_kind": "synthetic_contract",
|
|
"portfolio_id": "synthetic-pooled-detection",
|
|
"portfolio_lineage": {
|
|
"origin": "repository_fixture",
|
|
"source_path": "pooled.json",
|
|
"version": "1",
|
|
},
|
|
"split_roles": ["test"],
|
|
"selection_policy": "Fixed before evaluation; no selection.",
|
|
"claim_boundary": "Synthetic evaluator test; not product accuracy.",
|
|
"protected_policy": copy.deepcopy(EXPECTED_PROTECTED_POLICY),
|
|
"cases": [first, second],
|
|
}
|
|
path = tmp_path / "pooled.json"
|
|
path.write_text(json.dumps(portfolio, ensure_ascii=False), encoding="utf-8")
|
|
report = evaluate_cases(path, {"a-case", "b-case"})
|
|
|
|
case_ap = [item["metrics"]["ap50"] for item in report["results"]]
|
|
pooled = report["portfolio_metrics"]["object_detection"]["micro"]
|
|
expected_pooled = detection_ap(
|
|
[
|
|
{**item, "id": f"a-case::{item['id']}", "_sample_id": "a-case"}
|
|
for item in first["predictions"]
|
|
]
|
|
+ [
|
|
{**item, "id": f"b-case::{item['id']}", "_sample_id": "b-case"}
|
|
for item in second["predictions"]
|
|
],
|
|
[
|
|
{**item, "id": f"a-case::{item['id']}", "_sample_id": "a-case"}
|
|
for item in first["references"]
|
|
]
|
|
+ [
|
|
{**item, "id": f"b-case::{item['id']}", "_sample_id": "b-case"}
|
|
for item in second["references"]
|
|
],
|
|
0.5,
|
|
)
|
|
assert pooled["ap50"] == expected_pooled
|
|
assert pooled["ap50"] != pytest.approx(sum(case_ap) / len(case_ap))
|
|
assert sum(item["count"] for item in pooled["calibration"]["bins"]) == 3
|
|
subgroup = report["subgroups"]["dimensions"]["region"]["strata"]["flanders"]
|
|
subgroup_calibration = subgroup["task_metrics"]["object_detection"]["micro"][
|
|
"calibration"
|
|
]
|
|
assert sum(item["count"] for item in subgroup_calibration["bins"]) == 3
|
|
|
|
|
|
def test_raster_requires_exact_rectangular_alignment_masks_nodata_and_classes() -> None:
|
|
invalid_case = detection_case("invalid-class")
|
|
invalid_case["predictions"][0]["class"] = "road"
|
|
with pytest.raises(ValueError, match="outside the declared ontology"):
|
|
evaluate_object_detection(invalid_case)
|
|
|
|
valid = raster_case()
|
|
valid["predictions"][0][1] = -9999
|
|
valid["raster_context"]["prediction"]["mask"][0][1] = False
|
|
result = evaluate_raster_classification(valid)
|
|
assert result["metrics"]["prediction_coverage"] == pytest.approx(0.75)
|
|
assert result["metrics"]["per_class"]["1"]["false_negative"] == 1
|
|
|
|
jagged = raster_case("jagged")
|
|
jagged["predictions"][1].pop()
|
|
with pytest.raises(ValueError, match="exactly rectangular"):
|
|
evaluate_raster_classification(jagged)
|
|
|
|
missing_metadata = raster_case("missing-metadata")
|
|
del missing_metadata["raster_context"]["prediction"]["crs"]
|
|
with pytest.raises(ValueError, match="missing"):
|
|
evaluate_raster_classification(missing_metadata)
|
|
|
|
shifted = raster_case("shifted")
|
|
shifted["raster_context"]["prediction"]["transform"][2] += 1
|
|
with pytest.raises(ValueError, match="affine alignment differs"):
|
|
evaluate_raster_classification(shifted)
|
|
|
|
invalid_class = raster_case("invalid-class")
|
|
invalid_class["predictions"][0][0] = 3
|
|
with pytest.raises(ValueError, match="prediction class outside ontology"):
|
|
evaluate_raster_classification(invalid_class)
|
|
|
|
invalid_nodata = raster_case("invalid-nodata")
|
|
invalid_nodata["predictions"][0][0] = -9999
|
|
with pytest.raises(ValueError, match="marks nodata as valid"):
|
|
evaluate_raster_classification(invalid_nodata)
|
|
singular = raster_case("singular")
|
|
for side in ("reference", "prediction"):
|
|
singular["raster_context"][side]["transform"] = [1, 2, 0, 2, 4, 0]
|
|
with pytest.raises(ValueError, match="affine transform is singular"):
|
|
evaluate_raster_classification(singular)
|
|
|
|
|
|
def test_polygon_metrics_require_valid_geometry_projected_crs_and_metres() -> None:
|
|
assert evaluate_vector_comparison(polygon_case())["metrics"]["f1"] == 1.0
|
|
assert (
|
|
evaluate_footprint_segmentation(polygon_case("footprint_segmentation"))[
|
|
"metrics"
|
|
]["mean_iou"]
|
|
== 1.0
|
|
)
|
|
outer = [
|
|
[100000, 200000],
|
|
[100020, 200000],
|
|
[100020, 200020],
|
|
[100000, 200020],
|
|
[100000, 200000],
|
|
]
|
|
hole = [
|
|
[100005, 200005],
|
|
[100010, 200005],
|
|
[100010, 200010],
|
|
[100005, 200010],
|
|
[100005, 200005],
|
|
]
|
|
polygon_geometry = {"type": "Polygon", "coordinates": [outer, hole]}
|
|
geojson_polygon = polygon_case()
|
|
for side in ("references", "predictions"):
|
|
del geojson_polygon[side][0]["polygon"]
|
|
geojson_polygon[side][0]["geometry"] = copy.deepcopy(polygon_geometry)
|
|
polygon_result = evaluate_vector_comparison(geojson_polygon)
|
|
assert polygon_result["metrics"]["mean_iou"] == 1.0
|
|
assert polygon_result["raw"]["references"][0]["geometry"] == polygon_geometry
|
|
|
|
second = [
|
|
[100030, 200000],
|
|
[100040, 200000],
|
|
[100040, 200010],
|
|
[100030, 200010],
|
|
[100030, 200000],
|
|
]
|
|
multipolygon_geometry = {
|
|
"type": "MultiPolygon",
|
|
"coordinates": [[outer, hole], [second]],
|
|
}
|
|
geojson_multi = polygon_case()
|
|
for side in ("references", "predictions"):
|
|
del geojson_multi[side][0]["polygon"]
|
|
geojson_multi[side][0]["geometry"] = copy.deepcopy(multipolygon_geometry)
|
|
assert evaluate_vector_comparison(geojson_multi)["metrics"]["mean_iou"] == 1.0
|
|
|
|
geographic = polygon_case()
|
|
geographic["spatial_context"]["crs"] = "EPSG:4326"
|
|
with pytest.raises(ValueError, match="projected CRS"):
|
|
evaluate_vector_comparison(geographic)
|
|
mercator = polygon_case()
|
|
mercator["spatial_context"]["crs"] = "EPSG:3857"
|
|
with pytest.raises(ValueError, match="Mercator is unsuitable"):
|
|
evaluate_vector_comparison(mercator)
|
|
|
|
wrong_geography = polygon_case()
|
|
wrong_geography["spatial_context"]["crs"] = "EPSG:32660"
|
|
with pytest.raises(ValueError, match="does not overlap"):
|
|
evaluate_vector_comparison(wrong_geography)
|
|
|
|
wrong_units = polygon_case()
|
|
wrong_units["spatial_context"]["coordinate_units"] = "degree"
|
|
with pytest.raises(ValueError, match="must be 'm'"):
|
|
evaluate_vector_comparison(wrong_units)
|
|
|
|
bowtie = polygon_case()
|
|
bowtie["predictions"][0]["polygon"] = [
|
|
[100000, 200000],
|
|
[100010, 200010],
|
|
[100010, 200000],
|
|
[100000, 200010],
|
|
[100000, 200000],
|
|
]
|
|
with pytest.raises(ValueError, match="positive-area and valid"):
|
|
evaluate_vector_comparison(bowtie)
|
|
|
|
|
|
def test_failure_gallery_covers_geometry_raster_calibration_and_contexts() -> None:
|
|
segmentation = polygon_case("footprint_segmentation")
|
|
segmentation["predictions"][0]["polygon"] = [
|
|
[100000, 200000],
|
|
[100012, 200000],
|
|
[100012, 200010],
|
|
[100000, 200010],
|
|
[100000, 200000],
|
|
]
|
|
segmentation_result = evaluate_footprint_segmentation(segmentation)
|
|
segmentation_codes = {
|
|
item["error_code"] for item in segmentation_result["failures"]
|
|
}
|
|
assert {"M-BOUNDARY", "M-AREA-BIAS"} <= segmentation_codes
|
|
|
|
raster = raster_case("raster-taxonomy")
|
|
raster["metadata"]["tile_edge"] = True
|
|
raster["predictions"][0][0] = 1
|
|
raster_result = evaluate_raster_classification(raster)
|
|
raster_failure = next(
|
|
item
|
|
for item in raster_result["failures"]
|
|
if item["kind"] == "raster_misclassification"
|
|
)
|
|
assert raster_failure["error_code"] == "M-CLASS"
|
|
assert "tile_edge" in raster_failure["contexts"]
|
|
|
|
detection = detection_case("context-taxonomy")
|
|
detection["references"] = []
|
|
detection["predictions"] = [
|
|
{
|
|
"id": "high-confidence-fp",
|
|
"class": "building",
|
|
"bbox": [10, 10, 12, 12],
|
|
"confidence": 0.95,
|
|
}
|
|
]
|
|
detection["config"]["fixed_diagnostic_risk_thresholds"] = [0.5, 0.9]
|
|
detection["metadata"]["tile_edge"] = True
|
|
detection["metadata"]["ood"] = True
|
|
detection_result = evaluate_object_detection(detection)
|
|
false_positive = next(
|
|
item
|
|
for item in detection_result["failures"]
|
|
if item["kind"] == "false_positive"
|
|
)
|
|
assert {"tile_edge", "high_confidence", "out_of_distribution"} <= set(
|
|
false_positive["contexts"]
|
|
)
|
|
assert {"M-MISCALIBRATED", "M-OOD"} <= set(false_positive["secondary_error_codes"])
|
|
assert any(
|
|
item["error_code"] == "M-MISCALIBRATED" for item in detection_result["failures"]
|
|
)
|
|
|
|
|
|
def test_terrain_rejects_non_finite_and_validation_counts_only_critical_misses() -> (
|
|
None
|
|
):
|
|
terrain = {
|
|
"sample_id": "terrain",
|
|
"task": "terrain_interpretation",
|
|
"split": "test",
|
|
"metadata": copy.deepcopy(METADATA),
|
|
"config": {},
|
|
"lineage": lineage("terrain"),
|
|
"units": "m_TAW",
|
|
"references": [1.0, 2.0],
|
|
"predictions": [1.1, None],
|
|
}
|
|
assert evaluate_terrain(terrain)["metrics"]["coverage"] == 0.5
|
|
for field, value in (("references", math.nan), ("predictions", math.inf)):
|
|
invalid = copy.deepcopy(terrain)
|
|
invalid[field][0] = value
|
|
with pytest.raises(ValueError, match="finite number"):
|
|
evaluate_terrain(invalid)
|
|
|
|
validation = {
|
|
"sample_id": "validation",
|
|
"task": "geospatial_data_validation",
|
|
"split": "test",
|
|
"metadata": copy.deepcopy(METADATA),
|
|
"config": {},
|
|
"lineage": lineage("validation"),
|
|
"expected_anomalies": [{"code": "D-MAJOR", "severity": "major"}],
|
|
"observed_anomalies": [],
|
|
}
|
|
assert (
|
|
evaluate_validation(validation)["metrics"]["blocker_or_critical_miss_count"]
|
|
== 0
|
|
)
|
|
validation["expected_anomalies"].append(
|
|
{"code": "D-CRITICAL", "severity": "critical"}
|
|
)
|
|
assert (
|
|
evaluate_validation(validation)["metrics"]["blocker_or_critical_miss_count"]
|
|
== 1
|
|
)
|
|
validation["expected_anomalies"] = [{"code": "D-SEVERITY", "severity": "critical"}]
|
|
validation["observed_anomalies"] = [{"code": "D-SEVERITY", "severity": "minor"}]
|
|
severity_result = evaluate_validation(validation)
|
|
assert severity_result["metrics"]["true_positive"] == 0
|
|
assert severity_result["metrics"]["false_positive"] == 1
|
|
assert severity_result["metrics"]["false_negative"] == 1
|
|
assert severity_result["metrics"]["severity_mismatch_count"] == 1
|
|
assert severity_result["metrics"]["blocker_or_critical_miss_count"] == 1
|
|
|
|
validation["expected_anomalies"] = ["D-NO-SEVERITY"]
|
|
with pytest.raises(ValueError, match="include code and severity"):
|
|
evaluate_validation(validation)
|
|
|
|
|
|
def _subgroup_result(region: str, index: int, tp: int, fp: int, fn: int) -> dict:
|
|
metadata = copy.deepcopy(METADATA)
|
|
metadata["region"] = region
|
|
sample_id = f"{region}-{index}"
|
|
reference = {"id": "r", "class": "building", "bbox": [0, 0, 1, 1]}
|
|
prediction = {
|
|
"id": "p",
|
|
"class": "building",
|
|
"bbox": [0, 0, 1, 1],
|
|
"confidence": 0.8,
|
|
}
|
|
return {
|
|
"sample_id": sample_id,
|
|
"task": "object_detection",
|
|
"metadata": metadata,
|
|
"metrics": {**count_metrics(tp, fp, fn), "ap50": 0.5, "ap50_95": 0.4},
|
|
"raw": {
|
|
"sample_id": sample_id,
|
|
"classes": ["building"],
|
|
"references": [reference],
|
|
"predictions_pre_filter": [prediction],
|
|
"predictions_post_filter": [prediction],
|
|
"matches": [
|
|
{
|
|
"prediction_id": "p",
|
|
"reference_id": "r",
|
|
"overlap": 1.0,
|
|
"confidence": 0.8,
|
|
"class": "building",
|
|
}
|
|
],
|
|
},
|
|
"failures": [],
|
|
}
|
|
|
|
|
|
def test_subgroups_report_task_metrics_support_ci_and_worst_stratum() -> None:
|
|
results = [
|
|
*[_subgroup_result("strong", index, 10, 0, 0) for index in range(5)],
|
|
*[_subgroup_result("weak", index, 1, 4, 4) for index in range(5)],
|
|
]
|
|
report = subgroup_report(results)
|
|
region = report["dimensions"]["region"]
|
|
weak = region["strata"]["weak"]["task_metrics"]["object_detection"]
|
|
|
|
assert weak["status"] == "evaluable"
|
|
assert weak["case_support"] == 5
|
|
assert weak["micro"]["precision_ci95_wilson"]["status"] == "computed"
|
|
assert weak["macro"]["f1_case_support"] == 5
|
|
assert region["worst_stratum_by_task"]["object_detection"]["stratum"] == "weak"
|
|
|
|
insufficient = subgroup_report([_subgroup_result("thin", 0, 1, 0, 0)])
|
|
thin = insufficient["dimensions"]["region"]["strata"]["thin"]
|
|
assert thin["task_metrics"]["object_detection"]["status"] == "insufficient_support"
|
|
assert thin["release_gate_status"] == "not_evaluable"
|
|
assert insufficient["overall_status"] == "not_evaluable"
|
|
|
|
|
|
def test_capability_inventory_is_comprehensive_and_honest() -> None:
|
|
inventory = task_inventory()
|
|
assert {item["task"] for item in inventory} == TASKS
|
|
assert len(inventory) >= 15
|
|
assert all(item["implementation_paths"] for item in inventory)
|
|
assert all(item["suitable_metrics"] for item in inventory)
|
|
assistant = next(
|
|
item
|
|
for item in inventory
|
|
if item["capability_id"] == "geo_assistant_orchestration"
|
|
)
|
|
assert assistant["evaluation_status"].startswith("no_independent_accuracy_score")
|