Jens/geointel
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(accuracy): close phase 4 evidence bypasses

Browse Source
This commit is contained in:
Jens
2026-08-02 04:29:10 +02:00
parent bd2fd9780f
commit ee1982eff2
7 changed files with 5814 additions and 382 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/tests/test_accuracy_phase4_evaluator_hardening.py
+326 -5
View File
@@ -17,6 +17,7 @@ if str(SCRIPTS) not in sys.path:
from accuracy_phase4_evaluator import ( # noqa: E402
TASKS,
EXPECTED_PROTECTED_POLICY,
canonical_hash,
count_metrics,
detection_ap,
@@ -40,6 +41,7 @@ METADATA = {
"source": "synthetic-source",
"sensor": "synthetic-sensor",
"resolution_m": 0.25,
"context": "dense_urban",
"season": "summer",
"date": "2026-01-01",
"vegetation": "partial",
@@ -149,6 +151,7 @@ def test_raw_evidence_and_hashes_are_exact_and_recomputable(tmp_path: Path) -> N
case = detection_case()
portfolio = {
"schema_version": 2,
"portfolio_kind": "synthetic_contract",
"portfolio_id": "synthetic-hardening-test",
"portfolio_lineage": {
"origin": "repository_fixture",
@@ -158,7 +161,7 @@ def test_raw_evidence_and_hashes_are_exact_and_recomputable(tmp_path: Path) -> N
"split_roles": ["test"],
"selection_policy": "Fixed before evaluation; no selection.",
"claim_boundary": "Synthetic evaluator test; not product accuracy.",
"protected_policy": {"threshold_selection_allowed": False},
"protected_policy": copy.deepcopy(EXPECTED_PROTECTED_POLICY),
"cases": [case],
}
path = tmp_path / "portfolio.json"
@@ -203,6 +206,105 @@ def test_raw_evidence_and_hashes_are_exact_and_recomputable(tmp_path: Path) -> N
evaluate_cases(path, {case["sample_id"]})
def test_portfolio_schema_policy_metadata_and_lineage_are_strict(
tmp_path: Path,
) -> None:
case = detection_case("strict-contract")
portfolio = {
"schema_version": 2,
"portfolio_kind": "synthetic_contract",
"portfolio_id": "synthetic-strict-contract",
"portfolio_lineage": {
"origin": "repository_fixture",
"source_path": "synthetic.json",
"version": "1",
},
"split_roles": ["test"],
"selection_policy": "Fixed before evaluation; no selection.",
"claim_boundary": "Synthetic evaluator test; not product accuracy.",
"protected_policy": copy.deepcopy(EXPECTED_PROTECTED_POLICY),
"cases": [case],
}
path = tmp_path / "strict.json"
def evaluate(value: dict) -> dict:
path.write_text(json.dumps(value, ensure_ascii=False), encoding="utf-8")
return evaluate_cases(path, {case["sample_id"]})
assert evaluate(portfolio)["case_count"] == 1
for invalid_version in (1, True, "2"):
invalid = copy.deepcopy(portfolio)
invalid["schema_version"] = invalid_version
with pytest.raises(
ValueError, match="schema_version must be exactly integer 2"
):
evaluate(invalid)
invalid_policy = copy.deepcopy(portfolio)
invalid_policy["protected_policy"]["test_feedback_allowed"] = True
with pytest.raises(ValueError, match="protected_policy must exactly equal"):
evaluate(invalid_policy)
invalid_metadata = copy.deepcopy(portfolio)
invalid_metadata["cases"][0]["metadata"]["source"] = "unknown"
with pytest.raises(ValueError, match="metadata.source must be a meaningful"):
evaluate(invalid_metadata)
invalid_resolution = copy.deepcopy(portfolio)
invalid_resolution["cases"][0]["metadata"]["resolution_m"] = 0
with pytest.raises(ValueError, match="metadata.resolution_m must be positive"):
evaluate(invalid_resolution)
invalid_lineage = copy.deepcopy(portfolio)
del invalid_lineage["cases"][0]["lineage"]["prediction"]["derivation"]
with pytest.raises(ValueError, match="lineage.prediction missing"):
evaluate(invalid_lineage)
def test_portfolio_kind_separates_synthetic_and_governed_product_claims(
tmp_path: Path,
) -> None:
fixture_path = (
ROOT / "fixtures" / "accuracy" / "p4" / "protected-baseline-cases.json"
)
synthetic = json.loads(fixture_path.read_text(encoding="utf-8"))
allowed = {item["sample_id"] for item in synthetic["cases"]}
path = tmp_path / "portfolio.json"
missing_kind = copy.deepcopy(synthetic)
del missing_kind["portfolio_kind"]
path.write_text(json.dumps(missing_kind), encoding="utf-8")
with pytest.raises(ValueError, match="portfolio_kind"):
evaluate_cases(path, allowed)
confused = copy.deepcopy(synthetic)
confused["claim_boundary"] = "Governed product baseline accuracy evidence."
path.write_text(json.dumps(confused), encoding="utf-8")
with pytest.raises(ValueError, match="synthetic_contract"):
evaluate_cases(path, allowed)
governed = json.loads(
json.dumps(synthetic)
.replace("Synthetic", "Governed")
.replace("synthetic", "governed")
.replace("repository_fixture", "governed_product_evaluation")
)
governed["portfolio_kind"] = "governed_product_baseline"
governed["portfolio_id"] = "governed-product-baseline-test"
governed["claim_boundary"] = (
"Governed product baseline metrics recomputed from protected raw cases; "
"inference provenance is validated separately."
)
path.write_text(json.dumps(governed), encoding="utf-8")
report = evaluate_cases(path, allowed)
assert report["portfolio_kind"] == "governed_product_baseline"
assert set(report["evaluated_task_families"]) == TASKS
governed["cases"] = governed["cases"][:-1]
path.write_text(json.dumps(governed), encoding="utf-8")
def test_ap_ties_use_stable_ids_and_matching_is_class_aware() -> None:
references = [{"id": "r", "class": "building", "bbox": [0, 0, 4, 4]}]
predictions = [
@@ -233,6 +335,83 @@ def test_ap_ties_use_stable_ids_and_matching_is_class_aware() -> None:
assert detection_ap(wrong_class, references, 0.5) == pytest.approx(0.5)
def test_detection_ap_and_calibration_are_pooled_globally_and_per_subgroup(
tmp_path: Path,
) -> None:
first = detection_case("a-case")
first["predictions"] = [
{
"id": "p-true",
"class": "building",
"bbox": [0, 0, 4, 4],
"confidence": 0.9,
}
]
second = detection_case("b-case")
second["predictions"] = [
{
"id": "p-false",
"class": "building",
"bbox": [10, 10, 12, 12],
"confidence": 0.9,
},
{
"id": "p-true",
"class": "building",
"bbox": [0, 0, 4, 4],
"confidence": 0.8,
},
]
portfolio = {
"schema_version": 2,
"portfolio_kind": "synthetic_contract",
"portfolio_id": "synthetic-pooled-detection",
"portfolio_lineage": {
"origin": "repository_fixture",
"source_path": "pooled.json",
"version": "1",
},
"split_roles": ["test"],
"selection_policy": "Fixed before evaluation; no selection.",
"claim_boundary": "Synthetic evaluator test; not product accuracy.",
"protected_policy": copy.deepcopy(EXPECTED_PROTECTED_POLICY),
"cases": [first, second],
}
path = tmp_path / "pooled.json"
path.write_text(json.dumps(portfolio, ensure_ascii=False), encoding="utf-8")
report = evaluate_cases(path, {"a-case", "b-case"})
case_ap = [item["metrics"]["ap50"] for item in report["results"]]
pooled = report["portfolio_metrics"]["object_detection"]["micro"]
expected_pooled = detection_ap(
[
{**item, "id": f"a-case::{item['id']}", "_sample_id": "a-case"}
for item in first["predictions"]
]
+ [
{**item, "id": f"b-case::{item['id']}", "_sample_id": "b-case"}
for item in second["predictions"]
],
[
{**item, "id": f"a-case::{item['id']}", "_sample_id": "a-case"}
for item in first["references"]
]
+ [
{**item, "id": f"b-case::{item['id']}", "_sample_id": "b-case"}
for item in second["references"]
],
0.5,
)
assert pooled["ap50"] == expected_pooled
assert pooled["ap50"] != pytest.approx(sum(case_ap) / len(case_ap))
assert sum(item["count"] for item in pooled["calibration"]["bins"]) == 3
subgroup = report["subgroups"]["dimensions"]["region"]["strata"]["flanders"]
subgroup_calibration = subgroup["task_metrics"]["object_detection"]["micro"][
"calibration"
]
assert sum(item["count"] for item in subgroup_calibration["bins"]) == 3
def test_raster_requires_exact_rectangular_alignment_masks_nodata_and_classes() -> None:
invalid_case = detection_case("invalid-class")
invalid_case["predictions"][0]["class"] = "road"
@@ -270,6 +449,11 @@ def test_raster_requires_exact_rectangular_alignment_masks_nodata_and_classes()
invalid_nodata["predictions"][0][0] = -9999
with pytest.raises(ValueError, match="marks nodata as valid"):
evaluate_raster_classification(invalid_nodata)
singular = raster_case("singular")
for side in ("reference", "prediction"):
singular["raster_context"][side]["transform"] = [1, 2, 0, 2, 4, 0]
with pytest.raises(ValueError, match="affine transform is singular"):
evaluate_raster_classification(singular)
def test_polygon_metrics_require_valid_geometry_projected_crs_and_metres() -> None:
@@ -280,11 +464,59 @@ def test_polygon_metrics_require_valid_geometry_projected_crs_and_metres() -> No
]["mean_iou"]
== 1.0
)
outer = [
[100000, 200000],
[100020, 200000],
[100020, 200020],
[100000, 200020],
[100000, 200000],
]
hole = [
[100005, 200005],
[100010, 200005],
[100010, 200010],
[100005, 200010],
[100005, 200005],
]
polygon_geometry = {"type": "Polygon", "coordinates": [outer, hole]}
geojson_polygon = polygon_case()
for side in ("references", "predictions"):
del geojson_polygon[side][0]["polygon"]
geojson_polygon[side][0]["geometry"] = copy.deepcopy(polygon_geometry)
polygon_result = evaluate_vector_comparison(geojson_polygon)
assert polygon_result["metrics"]["mean_iou"] == 1.0
assert polygon_result["raw"]["references"][0]["geometry"] == polygon_geometry
second = [
[100030, 200000],
[100040, 200000],
[100040, 200010],
[100030, 200010],
[100030, 200000],
]
multipolygon_geometry = {
"type": "MultiPolygon",
"coordinates": [[outer, hole], [second]],
}
geojson_multi = polygon_case()
for side in ("references", "predictions"):
del geojson_multi[side][0]["polygon"]
geojson_multi[side][0]["geometry"] = copy.deepcopy(multipolygon_geometry)
assert evaluate_vector_comparison(geojson_multi)["metrics"]["mean_iou"] == 1.0
geographic = polygon_case()
geographic["spatial_context"]["crs"] = "EPSG:4326"
with pytest.raises(ValueError, match="projected CRS"):
evaluate_vector_comparison(geographic)
mercator = polygon_case()
mercator["spatial_context"]["crs"] = "EPSG:3857"
with pytest.raises(ValueError, match="Mercator is unsuitable"):
evaluate_vector_comparison(mercator)
wrong_geography = polygon_case()
wrong_geography["spatial_context"]["crs"] = "EPSG:32660"
with pytest.raises(ValueError, match="does not overlap"):
evaluate_vector_comparison(wrong_geography)
wrong_units = polygon_case()
wrong_units["spatial_context"]["coordinate_units"] = "degree"
@@ -303,6 +535,61 @@ def test_polygon_metrics_require_valid_geometry_projected_crs_and_metres() -> No
evaluate_vector_comparison(bowtie)
def test_failure_gallery_covers_geometry_raster_calibration_and_contexts() -> None:
segmentation = polygon_case("footprint_segmentation")
segmentation["predictions"][0]["polygon"] = [
[100000, 200000],
[100012, 200000],
[100012, 200010],
[100000, 200010],
[100000, 200000],
]
segmentation_result = evaluate_footprint_segmentation(segmentation)
segmentation_codes = {
item["error_code"] for item in segmentation_result["failures"]
}
assert {"M-BOUNDARY", "M-AREA-BIAS"} <= segmentation_codes
raster = raster_case("raster-taxonomy")
raster["metadata"]["tile_edge"] = True
raster["predictions"][0][0] = 1
raster_result = evaluate_raster_classification(raster)
raster_failure = next(
item
for item in raster_result["failures"]
if item["kind"] == "raster_misclassification"
)
assert raster_failure["error_code"] == "M-CLASS"
assert "tile_edge" in raster_failure["contexts"]
detection = detection_case("context-taxonomy")
detection["references"] = []
detection["predictions"] = [
{
"id": "high-confidence-fp",
"class": "building",
"bbox": [10, 10, 12, 12],
"confidence": 0.95,
}
]
detection["config"]["fixed_diagnostic_risk_thresholds"] = [0.5, 0.9]
detection["metadata"]["tile_edge"] = True
detection["metadata"]["ood"] = True
detection_result = evaluate_object_detection(detection)
false_positive = next(
item
for item in detection_result["failures"]
if item["kind"] == "false_positive"
)
assert {"tile_edge", "high_confidence", "out_of_distribution"} <= set(
false_positive["contexts"]
)
assert {"M-MISCALIBRATED", "M-OOD"} <= set(false_positive["secondary_error_codes"])
assert any(
item["error_code"] == "M-MISCALIBRATED" for item in detection_result["failures"]
)
def test_terrain_rejects_non_finite_and_validation_counts_only_critical_misses() -> (
None
):
@@ -345,26 +632,60 @@ def test_terrain_rejects_non_finite_and_validation_counts_only_critical_misses()
evaluate_validation(validation)["metrics"]["blocker_or_critical_miss_count"]
== 1
)
validation["expected_anomalies"] = [{"code": "D-SEVERITY", "severity": "critical"}]
validation["observed_anomalies"] = [{"code": "D-SEVERITY", "severity": "minor"}]
severity_result = evaluate_validation(validation)
assert severity_result["metrics"]["true_positive"] == 0
assert severity_result["metrics"]["false_positive"] == 1
assert severity_result["metrics"]["false_negative"] == 1
assert severity_result["metrics"]["severity_mismatch_count"] == 1
assert severity_result["metrics"]["blocker_or_critical_miss_count"] == 1
validation["expected_anomalies"] = ["D-NO-SEVERITY"]
with pytest.raises(ValueError, match="include code and severity"):
evaluate_validation(validation)
def _subgroup_result(region: str, tp: int, fp: int, fn: int) -> dict:
def _subgroup_result(region: str, index: int, tp: int, fp: int, fn: int) -> dict:
metadata = copy.deepcopy(METADATA)
metadata["region"] = region
sample_id = f"{region}-{index}"
reference = {"id": "r", "class": "building", "bbox": [0, 0, 1, 1]}
prediction = {
"id": "p",
"class": "building",
"bbox": [0, 0, 1, 1],
"confidence": 0.8,
}
return {
"sample_id": sample_id,
"task": "object_detection",
"metadata": metadata,
"metrics": {**count_metrics(tp, fp, fn), "ap50": 0.5, "ap50_95": 0.4},
"raw": {
"sample_id": sample_id,
"classes": ["building"],
"references": [reference],
"predictions_pre_filter": [prediction],
"predictions_post_filter": [prediction],
"matches": [
{
"prediction_id": "p",
"reference_id": "r",
"overlap": 1.0,
"confidence": 0.8,
"class": "building",
}
],
},
"failures": [],
}
def test_subgroups_report_task_metrics_support_ci_and_worst_stratum() -> None:
results = [
*[_subgroup_result("strong", 10, 0, 0) for _ in range(5)],
*[_subgroup_result("weak", 1, 4, 4) for _ in range(5)],
*[_subgroup_result("strong", index, 10, 0, 0) for index in range(5)],
*[_subgroup_result("weak", index, 1, 4, 4) for index in range(5)],
]
report = subgroup_report(results)
region = report["dimensions"]["region"]
@@ -376,7 +697,7 @@ def test_subgroups_report_task_metrics_support_ci_and_worst_stratum() -> None:
assert weak["macro"]["f1_case_support"] == 5
assert region["worst_stratum_by_task"]["object_detection"]["stratum"] == "weak"
insufficient = subgroup_report([_subgroup_result("thin", 1, 0, 0)])
insufficient = subgroup_report([_subgroup_result("thin", 0, 1, 0, 0)])
thin = insufficient["dimensions"]["region"]["strata"]["thin"]
assert thin["task_metrics"]["object_detection"]["status"] == "insufficient_support"
assert thin["release_gate_status"] == "not_evaluable"