Jens/geointel
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(accuracy): close phase 4 evidence bypasses

Browse Source
This commit is contained in:
Jens
2026-08-02 04:29:10 +02:00
parent bd2fd9780f
commit ee1982eff2
7 changed files with 5814 additions and 382 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/tests/test_accuracy_phase4_evaluation.py
+981 -12
View File
File diff suppressed because it is too large Load Diff
backend/tests/test_accuracy_phase4_evaluator_hardening.py
+326 -5
View File
@@ -17,6 +17,7 @@ if str(SCRIPTS) not in sys.path:
from accuracy_phase4_evaluator import ( # noqa: E402
TASKS,
EXPECTED_PROTECTED_POLICY,
canonical_hash,
count_metrics,
detection_ap,
@@ -40,6 +41,7 @@ METADATA = {
"source": "synthetic-source",
"sensor": "synthetic-sensor",
"resolution_m": 0.25,
"context": "dense_urban",
"season": "summer",
"date": "2026-01-01",
"vegetation": "partial",
@@ -149,6 +151,7 @@ def test_raw_evidence_and_hashes_are_exact_and_recomputable(tmp_path: Path) -> N
case = detection_case()
portfolio = {
"schema_version": 2,
"portfolio_kind": "synthetic_contract",
"portfolio_id": "synthetic-hardening-test",
"portfolio_lineage": {
"origin": "repository_fixture",
@@ -158,7 +161,7 @@ def test_raw_evidence_and_hashes_are_exact_and_recomputable(tmp_path: Path) -> N
"split_roles": ["test"],
"selection_policy": "Fixed before evaluation; no selection.",
"claim_boundary": "Synthetic evaluator test; not product accuracy.",
"protected_policy": {"threshold_selection_allowed": False},
"protected_policy": copy.deepcopy(EXPECTED_PROTECTED_POLICY),
"cases": [case],
}
path = tmp_path / "portfolio.json"
@@ -203,6 +206,105 @@ def test_raw_evidence_and_hashes_are_exact_and_recomputable(tmp_path: Path) -> N
evaluate_cases(path, {case["sample_id"]})
def test_portfolio_schema_policy_metadata_and_lineage_are_strict(
tmp_path: Path,
) -> None:
case = detection_case("strict-contract")
portfolio = {
"schema_version": 2,
"portfolio_kind": "synthetic_contract",
"portfolio_id": "synthetic-strict-contract",
"portfolio_lineage": {
"origin": "repository_fixture",
"source_path": "synthetic.json",
"version": "1",
},
"split_roles": ["test"],
"selection_policy": "Fixed before evaluation; no selection.",
"claim_boundary": "Synthetic evaluator test; not product accuracy.",
"protected_policy": copy.deepcopy(EXPECTED_PROTECTED_POLICY),
"cases": [case],
}
path = tmp_path / "strict.json"
def evaluate(value: dict) -> dict:
path.write_text(json.dumps(value, ensure_ascii=False), encoding="utf-8")
return evaluate_cases(path, {case["sample_id"]})
assert evaluate(portfolio)["case_count"] == 1
for invalid_version in (1, True, "2"):
invalid = copy.deepcopy(portfolio)
invalid["schema_version"] = invalid_version
with pytest.raises(
ValueError, match="schema_version must be exactly integer 2"
):
evaluate(invalid)
invalid_policy = copy.deepcopy(portfolio)
invalid_policy["protected_policy"]["test_feedback_allowed"] = True
with pytest.raises(ValueError, match="protected_policy must exactly equal"):
evaluate(invalid_policy)
invalid_metadata = copy.deepcopy(portfolio)
invalid_metadata["cases"][0]["metadata"]["source"] = "unknown"
with pytest.raises(ValueError, match="metadata.source must be a meaningful"):
evaluate(invalid_metadata)
invalid_resolution = copy.deepcopy(portfolio)
invalid_resolution["cases"][0]["metadata"]["resolution_m"] = 0
with pytest.raises(ValueError, match="metadata.resolution_m must be positive"):
evaluate(invalid_resolution)
invalid_lineage = copy.deepcopy(portfolio)
del invalid_lineage["cases"][0]["lineage"]["prediction"]["derivation"]
with pytest.raises(ValueError, match="lineage.prediction missing"):
evaluate(invalid_lineage)
def test_portfolio_kind_separates_synthetic_and_governed_product_claims(
tmp_path: Path,
) -> None:
fixture_path = (
ROOT / "fixtures" / "accuracy" / "p4" / "protected-baseline-cases.json"
)
synthetic = json.loads(fixture_path.read_text(encoding="utf-8"))
allowed = {item["sample_id"] for item in synthetic["cases"]}
path = tmp_path / "portfolio.json"
missing_kind = copy.deepcopy(synthetic)
del missing_kind["portfolio_kind"]
path.write_text(json.dumps(missing_kind), encoding="utf-8")
with pytest.raises(ValueError, match="portfolio_kind"):
evaluate_cases(path, allowed)
confused = copy.deepcopy(synthetic)
confused["claim_boundary"] = "Governed product baseline accuracy evidence."
path.write_text(json.dumps(confused), encoding="utf-8")
with pytest.raises(ValueError, match="synthetic_contract"):
evaluate_cases(path, allowed)
governed = json.loads(
json.dumps(synthetic)
.replace("Synthetic", "Governed")
.replace("synthetic", "governed")
.replace("repository_fixture", "governed_product_evaluation")
)
governed["portfolio_kind"] = "governed_product_baseline"
governed["portfolio_id"] = "governed-product-baseline-test"
governed["claim_boundary"] = (
"Governed product baseline metrics recomputed from protected raw cases; "
"inference provenance is validated separately."
)
path.write_text(json.dumps(governed), encoding="utf-8")
report = evaluate_cases(path, allowed)
assert report["portfolio_kind"] == "governed_product_baseline"
assert set(report["evaluated_task_families"]) == TASKS
governed["cases"] = governed["cases"][:-1]
path.write_text(json.dumps(governed), encoding="utf-8")
def test_ap_ties_use_stable_ids_and_matching_is_class_aware() -> None:
references = [{"id": "r", "class": "building", "bbox": [0, 0, 4, 4]}]
predictions = [
@@ -233,6 +335,83 @@ def test_ap_ties_use_stable_ids_and_matching_is_class_aware() -> None:
assert detection_ap(wrong_class, references, 0.5) == pytest.approx(0.5)
def test_detection_ap_and_calibration_are_pooled_globally_and_per_subgroup(
tmp_path: Path,
) -> None:
first = detection_case("a-case")
first["predictions"] = [
{
"id": "p-true",
"class": "building",
"bbox": [0, 0, 4, 4],
"confidence": 0.9,
}
]
second = detection_case("b-case")
second["predictions"] = [
{
"id": "p-false",
"class": "building",
"bbox": [10, 10, 12, 12],
"confidence": 0.9,
},
{
"id": "p-true",
"class": "building",
"bbox": [0, 0, 4, 4],
"confidence": 0.8,
},
]
portfolio = {
"schema_version": 2,
"portfolio_kind": "synthetic_contract",
"portfolio_id": "synthetic-pooled-detection",
"portfolio_lineage": {
"origin": "repository_fixture",
"source_path": "pooled.json",
"version": "1",
},
"split_roles": ["test"],
"selection_policy": "Fixed before evaluation; no selection.",
"claim_boundary": "Synthetic evaluator test; not product accuracy.",
"protected_policy": copy.deepcopy(EXPECTED_PROTECTED_POLICY),
"cases": [first, second],
}
path = tmp_path / "pooled.json"
path.write_text(json.dumps(portfolio, ensure_ascii=False), encoding="utf-8")
report = evaluate_cases(path, {"a-case", "b-case"})
case_ap = [item["metrics"]["ap50"] for item in report["results"]]
pooled = report["portfolio_metrics"]["object_detection"]["micro"]
expected_pooled = detection_ap(
[
{**item, "id": f"a-case::{item['id']}", "_sample_id": "a-case"}
for item in first["predictions"]
]
+ [
{**item, "id": f"b-case::{item['id']}", "_sample_id": "b-case"}
for item in second["predictions"]
],
[
{**item, "id": f"a-case::{item['id']}", "_sample_id": "a-case"}
for item in first["references"]
]
+ [
{**item, "id": f"b-case::{item['id']}", "_sample_id": "b-case"}
for item in second["references"]
],
0.5,
)
assert pooled["ap50"] == expected_pooled
assert pooled["ap50"] != pytest.approx(sum(case_ap) / len(case_ap))
assert sum(item["count"] for item in pooled["calibration"]["bins"]) == 3
subgroup = report["subgroups"]["dimensions"]["region"]["strata"]["flanders"]
subgroup_calibration = subgroup["task_metrics"]["object_detection"]["micro"][
"calibration"
]
assert sum(item["count"] for item in subgroup_calibration["bins"]) == 3
def test_raster_requires_exact_rectangular_alignment_masks_nodata_and_classes() -> None:
invalid_case = detection_case("invalid-class")
invalid_case["predictions"][0]["class"] = "road"
@@ -270,6 +449,11 @@ def test_raster_requires_exact_rectangular_alignment_masks_nodata_and_classes()
invalid_nodata["predictions"][0][0] = -9999
with pytest.raises(ValueError, match="marks nodata as valid"):
evaluate_raster_classification(invalid_nodata)
singular = raster_case("singular")
for side in ("reference", "prediction"):
singular["raster_context"][side]["transform"] = [1, 2, 0, 2, 4, 0]
with pytest.raises(ValueError, match="affine transform is singular"):
evaluate_raster_classification(singular)
def test_polygon_metrics_require_valid_geometry_projected_crs_and_metres() -> None:
@@ -280,11 +464,59 @@ def test_polygon_metrics_require_valid_geometry_projected_crs_and_metres() -> No
]["mean_iou"]
== 1.0
)
outer = [
[100000, 200000],
[100020, 200000],
[100020, 200020],
[100000, 200020],
[100000, 200000],
]
hole = [
[100005, 200005],
[100010, 200005],
[100010, 200010],
[100005, 200010],
[100005, 200005],
]
polygon_geometry = {"type": "Polygon", "coordinates": [outer, hole]}
geojson_polygon = polygon_case()
for side in ("references", "predictions"):
del geojson_polygon[side][0]["polygon"]
geojson_polygon[side][0]["geometry"] = copy.deepcopy(polygon_geometry)
polygon_result = evaluate_vector_comparison(geojson_polygon)
assert polygon_result["metrics"]["mean_iou"] == 1.0
assert polygon_result["raw"]["references"][0]["geometry"] == polygon_geometry
second = [
[100030, 200000],
[100040, 200000],
[100040, 200010],
[100030, 200010],
[100030, 200000],
]
multipolygon_geometry = {
"type": "MultiPolygon",
"coordinates": [[outer, hole], [second]],
}
geojson_multi = polygon_case()
for side in ("references", "predictions"):
del geojson_multi[side][0]["polygon"]
geojson_multi[side][0]["geometry"] = copy.deepcopy(multipolygon_geometry)
assert evaluate_vector_comparison(geojson_multi)["metrics"]["mean_iou"] == 1.0
geographic = polygon_case()
geographic["spatial_context"]["crs"] = "EPSG:4326"
with pytest.raises(ValueError, match="projected CRS"):
evaluate_vector_comparison(geographic)
mercator = polygon_case()
mercator["spatial_context"]["crs"] = "EPSG:3857"
with pytest.raises(ValueError, match="Mercator is unsuitable"):
evaluate_vector_comparison(mercator)
wrong_geography = polygon_case()
wrong_geography["spatial_context"]["crs"] = "EPSG:32660"
with pytest.raises(ValueError, match="does not overlap"):
evaluate_vector_comparison(wrong_geography)
wrong_units = polygon_case()
wrong_units["spatial_context"]["coordinate_units"] = "degree"
@@ -303,6 +535,61 @@ def test_polygon_metrics_require_valid_geometry_projected_crs_and_metres() -> No
evaluate_vector_comparison(bowtie)
def test_failure_gallery_covers_geometry_raster_calibration_and_contexts() -> None:
segmentation = polygon_case("footprint_segmentation")
segmentation["predictions"][0]["polygon"] = [
[100000, 200000],
[100012, 200000],
[100012, 200010],
[100000, 200010],
[100000, 200000],
]
segmentation_result = evaluate_footprint_segmentation(segmentation)
segmentation_codes = {
item["error_code"] for item in segmentation_result["failures"]
}
assert {"M-BOUNDARY", "M-AREA-BIAS"} <= segmentation_codes
raster = raster_case("raster-taxonomy")
raster["metadata"]["tile_edge"] = True
raster["predictions"][0][0] = 1
raster_result = evaluate_raster_classification(raster)
raster_failure = next(
item
for item in raster_result["failures"]
if item["kind"] == "raster_misclassification"
)
assert raster_failure["error_code"] == "M-CLASS"
assert "tile_edge" in raster_failure["contexts"]
detection = detection_case("context-taxonomy")
detection["references"] = []
detection["predictions"] = [
{
"id": "high-confidence-fp",
"class": "building",
"bbox": [10, 10, 12, 12],
"confidence": 0.95,
}
]
detection["config"]["fixed_diagnostic_risk_thresholds"] = [0.5, 0.9]
detection["metadata"]["tile_edge"] = True
detection["metadata"]["ood"] = True
detection_result = evaluate_object_detection(detection)
false_positive = next(
item
for item in detection_result["failures"]
if item["kind"] == "false_positive"
)
assert {"tile_edge", "high_confidence", "out_of_distribution"} <= set(
false_positive["contexts"]
)
assert {"M-MISCALIBRATED", "M-OOD"} <= set(false_positive["secondary_error_codes"])
assert any(
item["error_code"] == "M-MISCALIBRATED" for item in detection_result["failures"]
)
def test_terrain_rejects_non_finite_and_validation_counts_only_critical_misses() -> (
None
):
@@ -345,26 +632,60 @@ def test_terrain_rejects_non_finite_and_validation_counts_only_critical_misses()
evaluate_validation(validation)["metrics"]["blocker_or_critical_miss_count"]
== 1
)
validation["expected_anomalies"] = [{"code": "D-SEVERITY", "severity": "critical"}]
validation["observed_anomalies"] = [{"code": "D-SEVERITY", "severity": "minor"}]
severity_result = evaluate_validation(validation)
assert severity_result["metrics"]["true_positive"] == 0
assert severity_result["metrics"]["false_positive"] == 1
assert severity_result["metrics"]["false_negative"] == 1
assert severity_result["metrics"]["severity_mismatch_count"] == 1
assert severity_result["metrics"]["blocker_or_critical_miss_count"] == 1
validation["expected_anomalies"] = ["D-NO-SEVERITY"]
with pytest.raises(ValueError, match="include code and severity"):
evaluate_validation(validation)
def _subgroup_result(region: str, tp: int, fp: int, fn: int) -> dict:
def _subgroup_result(region: str, index: int, tp: int, fp: int, fn: int) -> dict:
metadata = copy.deepcopy(METADATA)
metadata["region"] = region
sample_id = f"{region}-{index}"
reference = {"id": "r", "class": "building", "bbox": [0, 0, 1, 1]}
prediction = {
"id": "p",
"class": "building",
"bbox": [0, 0, 1, 1],
"confidence": 0.8,
}
return {
"sample_id": sample_id,
"task": "object_detection",
"metadata": metadata,
"metrics": {**count_metrics(tp, fp, fn), "ap50": 0.5, "ap50_95": 0.4},
"raw": {
"sample_id": sample_id,
"classes": ["building"],
"references": [reference],
"predictions_pre_filter": [prediction],
"predictions_post_filter": [prediction],
"matches": [
{
"prediction_id": "p",
"reference_id": "r",
"overlap": 1.0,
"confidence": 0.8,
"class": "building",
}
],
},
"failures": [],
}
def test_subgroups_report_task_metrics_support_ci_and_worst_stratum() -> None:
results = [
*[_subgroup_result("strong", 10, 0, 0) for _ in range(5)],
*[_subgroup_result("weak", 1, 4, 4) for _ in range(5)],
*[_subgroup_result("strong", index, 10, 0, 0) for index in range(5)],
*[_subgroup_result("weak", index, 1, 4, 4) for index in range(5)],
]
report = subgroup_report(results)
region = report["dimensions"]["region"]
@@ -376,7 +697,7 @@ def test_subgroups_report_task_metrics_support_ci_and_worst_stratum() -> None:
assert weak["macro"]["f1_case_support"] == 5
assert region["worst_stratum_by_task"]["object_detection"]["stratum"] == "weak"
insufficient = subgroup_report([_subgroup_result("thin", 1, 0, 0)])
insufficient = subgroup_report([_subgroup_result("thin", 0, 1, 0, 0)])
thin = insufficient["dimensions"]["region"]["strata"]["thin"]
assert thin["task_metrics"]["object_detection"]["status"] == "insufficient_support"
assert thin["release_gate_status"] == "not_evaluable"
backend/tests/test_accuracy_phase4_split_hardening.py
+335 -19
View File
@@ -1,6 +1,7 @@
from __future__ import annotations
import copy
import hashlib
import json
import sys
from pathlib import Path
@@ -28,13 +29,25 @@ def load_source() -> dict:
return json.loads(SOURCE.read_text(encoding="utf-8"))
def build_fixture_manifests(source: dict) -> tuple[dict, dict, dict]:
return build_manifests(source, trusted_fixture_mode=True)
def assert_fixture_training_inputs_safe(
input_paths: list[Path], input_records: list[dict], protected: dict
) -> None:
assert_training_inputs_safe(
input_paths, input_records, protected, trusted_fixture_mode=True
)
def test_normative_roles_hashes_and_source_order_are_enforced() -> None:
source = load_source()
development, protected, leakage = build_manifests(source)
development, protected, leakage = build_fixture_manifests(source)
reversed_source = copy.deepcopy(source)
reversed_source["samples"].reverse()
reversed_development, reversed_protected, reversed_leakage = build_manifests(
reversed_source
reversed_development, reversed_protected, reversed_leakage = (
build_fixture_manifests(reversed_source)
)
assert leakage["status"] == "pass"
@@ -82,7 +95,7 @@ def test_cross_split_lineage_and_content_collisions_fail(
) -> None:
source = load_source()
source["samples"][8][field] = source["samples"][0][field]
_development, _protected, leakage = build_manifests(source)
_development, _protected, leakage = build_fixture_manifests(source)
assert leakage["status"] == "fail"
assert expected_code in {item["code"] for item in leakage["findings"]}
@@ -98,7 +111,7 @@ def test_cross_split_lineage_and_content_collisions_fail(
def test_near_duplicate_fingerprints_fail(field: str, expected_code: str) -> None:
source = load_source()
source["samples"][8][field] = source["samples"][0][field]
_development, _protected, leakage = build_manifests(source)
_development, _protected, leakage = build_fixture_manifests(source)
assert leakage["status"] == "fail"
assert expected_code in {item["code"] for item in leakage["findings"]}
@@ -111,7 +124,7 @@ def test_object_native_feature_and_spatial_collisions_fail() -> None:
"native_feature_ids"
]
source["samples"][10]["bbox"] = source["samples"][2]["bbox"]
_development, _protected, leakage = build_manifests(source)
_development, _protected, leakage = build_fixture_manifests(source)
codes = {item["code"] for item in leakage["findings"]}
assert {"S-OBJECT-INSTANCE", "S-NATIVE-FEATURE", "S-SPATIAL-OVERLAP"} <= codes
@@ -121,32 +134,34 @@ def test_non_metric_crs_and_missing_normative_role_fail_closed() -> None:
geographic = load_source()
geographic["crs"] = "EPSG:4326"
with pytest.raises(LeakageError, match="projected in metres"):
build_manifests(geographic)
build_fixture_manifests(geographic)
missing = load_source()
missing["samples"] = [
item for item in missing["samples"] if item["split"] != "calibration"
]
with pytest.raises(LeakageError, match="Required splits are absent"):
build_manifests(missing)
build_fixture_manifests(missing)
def test_training_firewall_only_allows_train_and_binds_protected_lineage() -> None:
development, protected, leakage = build_manifests(load_source())
development, protected, leakage = build_fixture_manifests(load_source())
assert leakage["status"] == "pass"
train = [item for item in development["samples"] if item["split"] == "train"]
validation = next(item for item in development["samples"] if item["split"] == "val")
protected_item = protected["samples"][0]
assert_training_inputs_safe([], train, protected)
assert_fixture_training_inputs_safe([], train, protected)
with pytest.raises(LeakageError, match="non_train_role"):
assert_training_inputs_safe([], [validation], protected)
assert_fixture_training_inputs_safe([], [validation], protected)
with pytest.raises(LeakageError, match="protected_identity"):
disguised = copy.deepcopy(train[0])
disguised["source_family"] = protected_item["source_family"]
assert_training_inputs_safe([], [disguised], protected)
assert_fixture_training_inputs_safe([], [disguised], protected)
with pytest.raises(LeakageError, match="protected_path"):
assert_training_inputs_safe([Path("vault/protected/test.json")], [], protected)
assert_fixture_training_inputs_safe(
[Path("vault/protected/test.json")], [], protected
)
def test_failed_generation_writes_status_but_no_consumable_manifests(
@@ -159,12 +174,18 @@ def test_failed_generation_writes_status_but_no_consumable_manifests(
output = tmp_path / "out"
with pytest.raises(LeakageError, match="Leakage gate failed"):
generate(source_path, output)
generate(source_path, output, trusted_fixture_mode=True)
status = json.loads((output / "generation-status.json").read_text(encoding="utf-8"))
assert status["status"] == "fail"
assert not (output / "development-split-manifest.json").exists()
assert not (output / "protected-split-manifest.json").exists()
assert status["consumable_manifests_valid"] is False
for name in (
"development-split-manifest.json",
"protected-split-manifest.json",
):
tombstone = json.loads((output / name).read_text(encoding="utf-8"))
assert tombstone["status"] == "invalidated"
assert tombstone["consumable"] is False
def test_deterministic_grouped_assignment_is_stable_and_keeps_relatives_together() -> (
@@ -196,11 +217,11 @@ def test_deterministic_grouped_assignment_is_stable_and_keeps_relatives_together
item.pop("split")
source["samples"][1]["group_id"] = source["samples"][0]["group_id"]
development, protected, leakage = build_manifests(source)
development, protected, leakage = build_fixture_manifests(source)
reversed_source = copy.deepcopy(source)
reversed_source["samples"].reverse()
reversed_development, reversed_protected, reversed_leakage = build_manifests(
reversed_source
reversed_development, reversed_protected, reversed_leakage = (
build_fixture_manifests(reversed_source)
)
assigned = {
@@ -220,3 +241,298 @@ def test_deterministic_grouped_assignment_is_stable_and_keeps_relatives_together
assert reversed_development["manifest_sha256"] == development["manifest_sha256"]
assert reversed_protected["manifest_sha256"] == protected["manifest_sha256"]
assert reversed_leakage == leakage
def test_source_cannot_weaken_mandatory_roles_or_policy_floors() -> None:
source = load_source()
source["required_splits"] = ["train", "val", "test"]
with pytest.raises(LeakageError, match="mandatory role order"):
build_fixture_manifests(source)
grouped = load_source()
grouped["assignment_mode"] = "deterministic_grouped"
grouped["split_assignment"] = {"roles": ["train", "val"]}
for item in grouped["samples"]:
item.pop("split")
with pytest.raises(LeakageError, match="mandatory role order"):
build_fixture_manifests(grouped)
for field, value in (
("independence_buffer_m", 1999),
("perceptual_hamming_threshold", 3),
("label_geometry_hamming_threshold", 1),
):
weakened = load_source()
weakened[field] = value
with pytest.raises(LeakageError, match="code-owned minimum"):
build_fixture_manifests(weakened)
def test_task_coverage_gap_and_even_justified_exemption_fail_honestly() -> None:
source = load_source()
source["samples"] = [
item
for item in source["samples"]
if item["sample_id"] != "validation-test-national"
]
_development, _protected, leakage = build_fixture_manifests(source)
assert leakage["status"] == "fail"
assert "S-PROTECTED-TASK-COVERAGE-MISSING" in {
finding["code"] for finding in leakage["findings"]
}
source["protected_task_exemptions"] = {
"geospatial_data_validation": (
"No evaluator-visible reference exists; challenge data remains sealed."
)
}
_development, _protected, leakage = build_fixture_manifests(source)
assert leakage["status"] == "fail"
assert "S-PROTECTED-TASK-COVERAGE-EXEMPTED" in {
finding["code"] for finding in leakage["findings"]
}
def test_identifiers_and_acquisition_dates_are_canonical_leakage_keys() -> None:
source = load_source()
source["samples"][8]["group_id"] = " G01 "
_development, _protected, leakage = build_fixture_manifests(source)
assert "S-SPATIAL-GROUP" in {item["code"] for item in leakage["findings"]}
temporal = load_source()
temporal["samples"][8]["acquisition_date"] = temporal["samples"][0][
"acquisition_date"
]
_development, _protected, leakage = build_fixture_manifests(temporal)
assert "S-ACQUISITION-DATE" in {item["code"] for item in leakage["findings"]}
ambiguous = load_source()
ambiguous["samples"][8]["sample_id"] = " DET-TRAIN-A "
with pytest.raises(LeakageError, match="ambiguous canonical sample_id"):
build_fixture_manifests(ambiguous)
def test_challenge_is_sealed_in_standard_manifest() -> None:
_development, protected, leakage = build_fixture_manifests(load_source())
assert leakage["status"] == "pass"
challenge = [item for item in protected["samples"] if item["split"] == "challenge"]
forbidden = {
"label_sha256",
"label_geometry_hash",
"label_geometry_fingerprint",
"object_ids",
"native_feature_ids",
"record_sha256",
"label_path",
"label_geometry_path",
}
assert challenge
assert all(item["sealed"] is True for item in challenge)
assert all(not (forbidden & set(item)) for item in challenge)
def test_firewall_rejects_empty_tampered_wrong_and_renamed_manifests(
tmp_path: Path,
) -> None:
development, protected, leakage = build_fixture_manifests(load_source())
assert leakage["status"] == "pass"
train = [item for item in development["samples"] if item["split"] == "train"]
with pytest.raises(LeakageError, match="empty manifest"):
assert_fixture_training_inputs_safe([], train, {})
with pytest.raises(LeakageError, match="missing fields"):
assert_fixture_training_inputs_safe([], train, development)
tampered = copy.deepcopy(protected)
tampered["samples"].pop()
with pytest.raises(LeakageError, match="checksum mismatch"):
assert_fixture_training_inputs_safe([], train, tampered)
renamed = tmp_path / "ordinary-training-input.json"
renamed.write_text(json.dumps(protected), encoding="utf-8")
with pytest.raises(LeakageError, match="protected_manifest_content"):
assert_fixture_training_inputs_safe([renamed], train, protected)
def _canonical_json_sha256(value: object) -> str:
return hashlib.sha256(
json.dumps(
value, ensure_ascii=False, sort_keys=True, separators=(",", ":")
).encode("utf-8")
).hexdigest()
def make_governed_source(tmp_path: Path) -> dict:
source = load_source()
source["dataset_version"] = "governed-production-v1"
source["claim_boundary"] = "Governed production split source."
p3_items: list[dict] = []
provenance_records: list[dict] = []
asset_fields = {
"raw_image": ("raw_image_path", "raw_image_sha256"),
"processed_image": ("processed_image_path", "processed_image_sha256"),
"label": ("label_path", "label_sha256"),
"label_geometry": ("label_geometry_path", "label_geometry_hash"),
}
for sample in source["samples"]:
assets: dict[str, dict] = {}
p3_ids: dict[str, str] = {}
for role, (path_field, hash_field) in asset_fields.items():
relative = Path("assets") / sample["sample_id"] / f"{role}.bin"
path = tmp_path / relative
path.parent.mkdir(parents=True, exist_ok=True)
path.write_bytes(f"{sample['sample_id']}:{role}:governed".encode())
digest = hashlib.sha256(path.read_bytes()).hexdigest()
p3_id = hashlib.sha256(
f"{sample['sample_id']}:{role}".encode()
).hexdigest()[:20]
relative_posix = relative.as_posix()
sample[path_field] = relative_posix
sample[hash_field] = digest
p3_ids[role] = p3_id
assets[role] = {
"path": relative_posix,
"sha256": digest,
"size_bytes": path.stat().st_size,
"p3_item_id": p3_id,
}
p3_items.append(
{
"item_id": p3_id,
"path": relative_posix,
"sha256": digest,
"size_bytes": path.stat().st_size,
"status": "examined",
"read_status": "readable",
"recommended_action": "accept",
"empty_content": False,
"schema_conformity": "conformant",
"anomalies": [],
}
)
provenance_id = (
"prov-" + hashlib.sha256(sample["sample_id"].encode()).hexdigest()[:24]
)
sample["governance_binding"] = {
"source_provenance_record_id": provenance_id,
"p3_item_ids": p3_ids,
}
provenance_records.append(
{
"record_id": provenance_id,
"sample_id": sample["sample_id"],
"status": "accepted",
"lineage_status": "complete",
"training_allowed": True,
"perceptual_image_hash": sample["perceptual_image_hash"],
"label_geometry_fingerprint": sample["label_geometry_fingerprint"],
"assets": assets,
}
)
p3 = {
"schema_version": 1,
"scan_id": "p3-test-governed",
"scanner_version": "3.0.3",
"completed_at": "2026-08-02T12:00:00+02:00",
"items": p3_items,
"reconciliation": {
"examined": len(p3_items),
"skipped": 0,
"unreachable": 0,
"inventory_total": len(p3_items),
"reconciles": True,
},
}
provenance = {
"schema_version": 1,
"manifest_type": "geointel_phase4_source_provenance",
"status": "pass",
"records": provenance_records,
"records_canonical_json_sha256": _canonical_json_sha256(provenance_records),
}
p3_path = tmp_path / "p3.json"
provenance_path = tmp_path / "provenance.json"
p3_path.write_text(json.dumps(p3), encoding="utf-8")
provenance_path.write_text(json.dumps(provenance), encoding="utf-8")
source["governance_evidence"] = {
"p3_scan_manifest": {
"path": p3_path.name,
"sha256": hashlib.sha256(p3_path.read_bytes()).hexdigest(),
},
"source_provenance_manifest": {
"path": provenance_path.name,
"sha256": hashlib.sha256(provenance_path.read_bytes()).hexdigest(),
},
}
return source
def test_fixture_mode_is_explicit_and_source_metadata_cannot_enable_it() -> None:
with pytest.raises(LeakageError, match="explicit trusted_fixture_mode"):
build_manifests(load_source())
source = load_source()
source["trusted_fixture_mode"] = True
with pytest.raises(LeakageError, match="cannot be enabled by source metadata"):
build_manifests(source, trusted_fixture_mode=True)
def test_empty_or_arbitrary_governance_json_is_rejected(tmp_path: Path) -> None:
source = load_source()
source["dataset_version"] = "governed-production-v1"
source["claim_boundary"] = "Governed production split source."
p3 = tmp_path / "p3.json"
provenance = tmp_path / "provenance.json"
p3.write_text("{}", encoding="utf-8")
provenance.write_text("{}", encoding="utf-8")
source["governance_evidence"] = {
"p3_scan_manifest": {
"path": p3.name,
"sha256": hashlib.sha256(p3.read_bytes()).hexdigest(),
},
"source_provenance_manifest": {
"path": provenance.name,
"sha256": hashlib.sha256(provenance.read_bytes()).hexdigest(),
},
}
with pytest.raises(LeakageError, match="non-empty JSON object"):
build_manifests(source, source_root=tmp_path)
def test_governed_records_require_exact_provenance_paths_and_live_bytes(
tmp_path: Path,
) -> None:
source = make_governed_source(tmp_path)
development, protected, leakage = build_manifests(source, source_root=tmp_path)
assert leakage["status"] == "pass"
assert protected["source_trust"]["production_accuracy_use_allowed"] is True
train = [item for item in development["samples"] if item["split"] == "train"]
assert_training_inputs_safe([], train, protected)
no_paths = copy.deepcopy(train[0])
no_paths.pop("content_path_bindings")
with pytest.raises(LeakageError, match="missing_accessible_content_paths"):
assert_training_inputs_safe([], [no_paths], protected)
relabeled = copy.deepcopy(
next(item for item in protected["samples"] if item["split"] == "test")
)
relabeled["split"] = "train"
relabeled.pop("content_path_bindings")
for field in ("sample_id", "group_id", "source_family", "temporal_family"):
relabeled[field] = f"spoofed-{field}"
with pytest.raises(
LeakageError, match="unavailable_provenance_record|protected_identity"
):
assert_training_inputs_safe([], [relabeled], protected)
broken_binding = copy.deepcopy(source)
broken_binding["samples"][0]["governance_binding"]["p3_item_ids"]["raw_image"] = (
"0" * 20
)
with pytest.raises(LeakageError, match="provenance binding mismatch|P3 record"):
build_manifests(broken_binding, source_root=tmp_path)
raw_path = Path(train[0]["content_path_bindings"]["raw_image"]["resolved_path"])
raw_path.write_bytes(b"mutated after manifest creation")
with pytest.raises(LeakageError, match="record_path_hash_binding_mismatch"):
assert_training_inputs_safe([], train, protected)
fixtures/accuracy/p4/protected-baseline-cases.json
+1
View File
@@ -1,5 +1,6 @@
{
"schema_version": 2,
"portfolio_kind": "synthetic_contract",
"portfolio_id": "geointel-p4-reference-harness-v2",
"split_roles": [
"test",
scripts/accuracy_phase4_evaluator.py
+567 -75
View File
@@ -1,7 +1,8 @@
"""Fail-closed, task-aware metrics for the GeoIntel Phase 4 benchmark.
The bundled portfolio is a synthetic contract fixture. Nothing emitted by
this module is, by itself, evidence of production-model accuracy.
The evaluator accepts explicitly separated synthetic contract portfolios and
governed product-baseline portfolios. Nothing emitted by this module is, by
itself, evidence that inference provenance or production release gates passed.
"""
from __future__ import annotations
@@ -14,7 +15,8 @@ from pathlib import Path
from statistics import mean, median
from typing import Any, Callable, Iterable
EVALUATOR_VERSION = "2.0.0"
PORTFOLIO_KINDS = {"synthetic_contract", "governed_product_baseline"}
EVALUATOR_VERSION = "2.1.0"
REPORT_SCHEMA_VERSION = 2
SUBGROUP_MIN_CASE_SUPPORT = 5
CANONICAL_JSON_SPEC = (
@@ -46,6 +48,54 @@ ERROR_CODES = {
"validation_false_positive": "D-VALIDATION-FP",
"validation_false_negative": "D-VALIDATION-FN",
"terrain_missing": "P-PARTIAL",
"raster_misclassification": "M-CLASS",
"raster_missing": "P-PARTIAL",
"boundary_error": "M-BOUNDARY",
"area_bias": "M-AREA-BIAS",
"miscalibrated": "M-MISCALIBRATED",
}
EXPECTED_PROTECTED_POLICY = {
"operating_point_selection_allowed": False,
"diagnostic_curves_select_operating_point": False,
"test_feedback_allowed": False,
"threshold_selection_source": "pre_registered_configuration_only",
}
REQUIRED_METADATA_STRING_FIELDS = (
"region",
"municipality",
"urbanity",
"object_size",
"source",
"sensor",
"season",
"date",
"vegetation",
"occlusion",
"difficulty",
"context",
)
LINEAGE_SIDES = ("reference", "prediction")
LINEAGE_FIELDS = ("source_id", "source_version", "derivation")
NON_MEANINGFUL_TOKENS = {"", "unknown", "n/a", "na", "null", "tbd", "todo"}
BELGIUM_SCOPE_BOUNDS = (1.9, 49.4, 7.5, 52.1)
FAILURE_TAXONOMY = {
"M-FP-CONFUSER": "unmatched object or event prediction",
"M-FN-MISSED": "unmatched reference object or event",
"M-CLASS": "raster class differs from the valid reference class",
"M-BOUNDARY": "matched footprint has a measurable boundary deviation",
"M-AREA-BIAS": "matched footprint has a measurable relative area bias",
"M-MISCALIBRATED": "confidence diagnostic deviates from observed correctness",
"M-OOD": "error observed in an explicitly declared out-of-distribution case",
"D-VALIDATION-FP": "anomaly or severity reported without an exact reference match",
"D-VALIDATION-FN": "reference anomaly or severity not exactly reported",
"P-PARTIAL": "required prediction value is unavailable",
}
FAILURE_CONTEXTS = {
"tile_edge": "case metadata explicitly marks tile-edge context",
"high_confidence": (
"false positive meets the highest frozen diagnostic confidence threshold"
),
"out_of_distribution": "case metadata explicitly marks OOD context",
}
@@ -146,17 +196,62 @@ def _nonempty_mapping(value: Any, label: str) -> dict[str, Any]:
return value
def _meaningful_string(
value: Any,
label: str,
*,
allow_not_applicable: bool = False,
) -> str:
if not isinstance(value, str) or value.strip().lower() in NON_MEANINGFUL_TOKENS:
raise ValueError(f"{label} must be a meaningful non-empty string")
normalized = value.strip()
if not allow_not_applicable and normalized.lower() == "not_applicable":
raise ValueError(f"{label} may not be not_applicable")
return normalized
def _validate_lineage(case: dict[str, Any]) -> None:
sample_id = case["sample_id"]
lineage = _nonempty_mapping(case.get("lineage"), f"{sample_id}: lineage")
missing_sides = sorted(set(LINEAGE_SIDES) - lineage.keys())
if missing_sides:
raise ValueError(f"{sample_id}: lineage missing {missing_sides}")
for side in LINEAGE_SIDES:
entry = _nonempty_mapping(lineage.get(side), f"{sample_id}: lineage.{side}")
missing_fields = sorted(set(LINEAGE_FIELDS) - entry.keys())
if missing_fields:
raise ValueError(f"{sample_id}: lineage.{side} missing {missing_fields}")
for field in LINEAGE_FIELDS:
_meaningful_string(
entry.get(field),
f"{sample_id}: lineage.{side}.{field}",
)
def _validate_case_common(case: dict[str, Any]) -> None:
sample_id = case.get("sample_id")
if not isinstance(sample_id, str) or not sample_id.strip():
raise ValueError("Every case requires a non-empty sample_id")
if case.get("task") not in TASKS:
raise ValueError(f"{sample_id}: unsupported task {case.get('task')!r}")
if not isinstance(case.get("metadata"), dict):
raise ValueError(f"{sample_id}: metadata must be an object")
metadata = _nonempty_mapping(case.get("metadata"), f"{sample_id}: metadata")
missing_metadata = sorted(set(REQUIRED_METADATA_STRING_FIELDS) - metadata.keys())
if missing_metadata:
raise ValueError(f"{sample_id}: metadata missing {missing_metadata}")
for field in REQUIRED_METADATA_STRING_FIELDS:
_meaningful_string(
metadata.get(field),
f"{sample_id}: metadata.{field}",
allow_not_applicable=True,
)
resolution = metadata.get("resolution_m")
if case["task"] == "geospatial_data_validation" and resolution is None:
pass
elif _finite_float(resolution, f"{sample_id}: metadata.resolution_m") <= 0:
raise ValueError(f"{sample_id}: metadata.resolution_m must be positive")
if not isinstance(case.get("config"), dict):
raise ValueError(f"{sample_id}: config must be an explicit object")
_nonempty_mapping(case.get("lineage"), f"{sample_id}: lineage")
_validate_lineage(case)
def _stable_id(item: dict[str, Any], label: str) -> str:
@@ -339,6 +434,8 @@ def detection_ap(
reference = references[index]
if not _class_compatible(prediction, reference):
continue
if prediction.get("_sample_id") != reference.get("_sample_id"):
continue
candidates.append(
(
bbox_iou(prediction["bbox"], reference["bbox"]),
@@ -510,6 +607,18 @@ def evaluate_object_detection(case: dict[str, Any]) -> dict[str, Any]:
case["config"].get("match_iou"),
f"{case['sample_id']}.config.match_iou",
)
diagnostic_thresholds = case["config"].get("fixed_diagnostic_risk_thresholds")
if diagnostic_thresholds is not None:
if not isinstance(diagnostic_thresholds, list) or not diagnostic_thresholds:
raise ValueError(
f"{case['sample_id']}: fixed diagnostic risk thresholds must be a "
"non-empty list"
)
for index, value in enumerate(diagnostic_thresholds):
_probability(
value,
f"{case['sample_id']}.config.fixed_diagnostic_risk_thresholds[{index}]",
)
retained = [item for item in predictions if float(item["confidence"]) >= threshold]
matches, false_positives, false_negatives = greedy_match(
retained,
@@ -538,6 +647,10 @@ def evaluate_object_detection(case: dict[str, Any]) -> dict[str, Any]:
class_metrics["ap50"] = detection_ap(
all_class_predictions, class_references, 0.5
)
class_metrics["ap50_95"] = _mean_or_none(
detection_ap(all_class_predictions, class_references, 0.5 + step * 0.05)
for step in range(10)
)
per_class[str(class_value)] = class_metrics
metrics = count_metrics(len(matches), len(false_positives), len(false_negatives))
metrics.update(
@@ -555,9 +668,11 @@ def evaluate_object_detection(case: dict[str, Any]) -> dict[str, Any]:
"coverage_risk": coverage_risk(
predictions, references, match_iou, threshold
),
"map50": _mean_or_none(item["ap50"] for item in per_class.values()),
"map50_95": _mean_or_none(item["ap50_95"] for item in per_class.values()),
}
)
return result(
evaluation = result(
case,
metrics,
matches,
@@ -571,12 +686,24 @@ def evaluate_object_detection(case: dict[str, Any]) -> dict[str, Any]:
"threshold": threshold,
},
)
def polygon(item: dict[str, Any]):
from shapely.geometry import Polygon
return Polygon(item["polygon"])
calibration = metrics["calibration"]
if calibration["status"] == "computed" and calibration["ece"] > 1e-12:
evaluation["failures"].append(
failure_entry(
case,
"miscalibrated",
{
"ece": calibration["ece"],
"brier": calibration["brier"],
"prediction_support": len(retained),
"claim_boundary": (
"diagnostic deviation only; not a representative "
"population-calibration claim"
),
},
)
)
return evaluation
def distribution(values: list[float]) -> dict[str, Any]:
@@ -594,6 +721,18 @@ def distribution(values: list[float]) -> dict[str, Any]:
}
def _bounds_overlap(
left: tuple[float, float, float, float],
right: tuple[float, float, float, float],
) -> bool:
return not (
left[2] < right[0]
or left[0] > right[2]
or left[3] < right[1]
or left[1] > right[3]
)
def _validate_metric_spatial_context(case: dict[str, Any]) -> dict[str, Any]:
context = _nonempty_mapping(
case.get("spatial_context"),
@@ -629,32 +768,99 @@ def _validate_metric_spatial_context(case: dict[str, Any]) -> dict[str, Any]:
for axis in crs.axis_info[:2]
):
raise ValueError(f"{case['sample_id']}: CRS axes must use metres")
if crs.to_epsg() in {3395, 3857}:
raise ValueError(
f"{case['sample_id']}: Web/World Mercator is unsuitable for "
"benchmark area, boundary and distance metrics"
)
area = crs.area_of_use
if area is None:
raise ValueError(
f"{case['sample_id']}: CRS area of use is unavailable; Belgian "
"metric suitability cannot be proven"
)
crs_bounds = (area.west, area.south, area.east, area.north)
if not _bounds_overlap(crs_bounds, BELGIUM_SCOPE_BOUNDS):
raise ValueError(
f"{case['sample_id']}: CRS area of use does not overlap the "
"declared Belgium and Belgian North Sea product scope"
)
return context
def _validated_polygon(item: dict[str, Any], label: str):
coordinates = item.get("polygon")
def _validated_ring(coordinates: Any, label: str) -> list[list[float]]:
if not isinstance(coordinates, list) or len(coordinates) < 4:
raise ValueError(
f"{label}.polygon must contain a closed ring with at least four points"
f"{label} must contain a closed ring with at least four points"
)
normalized: list[tuple[float, float]] = []
normalized: list[list[float]] = []
for index, point in enumerate(coordinates):
if not isinstance(point, list) or len(point) != 2:
raise ValueError(f"{label}.polygon[{index}] must be an [x, y] pair")
raise ValueError(f"{label}[{index}] must be an [x, y] pair")
normalized.append(
(
_finite_float(point[0], f"{label}.polygon[{index}][0]"),
_finite_float(point[1], f"{label}.polygon[{index}][1]"),
)
[
_finite_float(point[0], f"{label}[{index}][0]"),
_finite_float(point[1], f"{label}[{index}][1]"),
]
)
if normalized[0] != normalized[-1]:
raise ValueError(f"{label}.polygon ring must be explicitly closed")
from shapely.geometry import Polygon
raise ValueError(f"{label} must be explicitly closed")
return normalized
geometry = Polygon(normalized)
def _validated_polygon(item: dict[str, Any], label: str):
if "geometry" in item and "polygon" in item:
raise ValueError(f"{label} may not declare both geometry and polygon")
if "geometry" in item:
payload = item["geometry"]
if not isinstance(payload, dict):
raise ValueError(f"{label}.geometry must be a GeoJSON object")
geometry_type = payload.get("type")
coordinates = payload.get("coordinates")
if geometry_type == "Polygon":
if not isinstance(coordinates, list) or not coordinates:
raise ValueError(f"{label}.geometry Polygon requires rings")
normalized_coordinates: Any = [
_validated_ring(ring, f"{label}.geometry.coordinates[{index}]")
for index, ring in enumerate(coordinates)
]
elif geometry_type == "MultiPolygon":
if not isinstance(coordinates, list) or not coordinates:
raise ValueError(
f"{label}.geometry MultiPolygon requires polygon members"
)
normalized_coordinates = []
for polygon_index, polygon_coordinates in enumerate(coordinates):
if not isinstance(polygon_coordinates, list) or not polygon_coordinates:
raise ValueError(
f"{label}.geometry.coordinates[{polygon_index}] requires rings"
)
normalized_coordinates.append(
[
_validated_ring(
ring,
f"{label}.geometry.coordinates[{polygon_index}]"
f"[{ring_index}]",
)
for ring_index, ring in enumerate(polygon_coordinates)
]
)
else:
raise ValueError(f"{label}.geometry.type must be Polygon or MultiPolygon")
normalized_payload = {
"type": geometry_type,
"coordinates": normalized_coordinates,
}
else:
normalized_payload = {
"type": "Polygon",
"coordinates": [_validated_ring(item.get("polygon"), f"{label}.polygon")],
}
from shapely.geometry import shape
geometry = shape(normalized_payload)
if geometry.is_empty or geometry.area <= 0 or not geometry.is_valid:
raise ValueError(f"{label}.polygon must be non-empty, positive-area and valid")
raise ValueError(f"{label}.geometry must be non-empty, positive-area and valid")
return geometry
@@ -679,16 +885,14 @@ def _validated_polygon_items(
validated_references = [
dict(
item,
geometry=_validated_polygon(
item, f"{case['sample_id']}.references[{index}]"
),
_shape=_validated_polygon(item, f"{case['sample_id']}.references[{index}]"),
)
for index, item in enumerate(references)
]
validated_predictions = [
dict(
item,
geometry=_validated_polygon(
_shape=_validated_polygon(
item, f"{case['sample_id']}.predictions[{index}]"
),
)
@@ -698,8 +902,8 @@ def _validated_polygon_items(
def _polygon_overlap(prediction: dict[str, Any], reference: dict[str, Any]) -> float:
intersection = prediction["geometry"].intersection(reference["geometry"]).area
union = prediction["geometry"].union(reference["geometry"]).area
intersection = prediction["_shape"].intersection(reference["_shape"]).area
union = prediction["_shape"].union(reference["_shape"]).area
return intersection / union if union > 0 else 0.0
@@ -753,13 +957,23 @@ def evaluate_footprint_segmentation(case: dict[str, Any]) -> dict[str, Any]:
centroids = []
area_errors = []
for match in matches:
pred = by_prediction[match["prediction_id"]]["geometry"]
ref = by_reference[match["reference_id"]]["geometry"]
pred = by_prediction[match["prediction_id"]]["_shape"]
ref = by_reference[match["reference_id"]]["_shape"]
intersection = pred.intersection(ref).area
dice.append(2 * intersection / (pred.area + ref.area))
boundary.append(boundary_f1(pred, ref, tolerance))
centroids.append(pred.centroid.distance(ref.centroid))
area_errors.append((pred.area - ref.area) / ref.area)
dice_value = 2 * intersection / (pred.area + ref.area)
boundary_value = boundary_f1(pred, ref, tolerance)
centroid_value = pred.centroid.distance(ref.centroid)
area_error = (pred.area - ref.area) / ref.area
dice.append(dice_value)
boundary.append(boundary_value)
centroids.append(centroid_value)
area_errors.append(area_error)
match.update(
dice=dice_value,
boundary_f1=boundary_value,
centroid_distance_m=centroid_value,
relative_area_error=area_error,
)
metrics = count_metrics(len(matches), len(false_positives), len(false_negatives))
metrics.update(
{
@@ -773,13 +987,40 @@ def evaluate_footprint_segmentation(case: dict[str, Any]) -> dict[str, Any]:
"spatial_context_validated": True,
}
)
return result(
evaluation = result(
case,
metrics,
matches,
false_positives,
false_negatives,
)
for match in matches:
if match["boundary_f1"] is not None and match["boundary_f1"] < 1 - 1e-12:
evaluation["failures"].append(
failure_entry(
case,
"boundary_error",
{
"prediction_id": match["prediction_id"],
"reference_id": match["reference_id"],
"boundary_f1": match["boundary_f1"],
"boundary_tolerance_m": tolerance,
},
)
)
if abs(match["relative_area_error"]) > 1e-12:
evaluation["failures"].append(
failure_entry(
case,
"area_bias",
{
"prediction_id": match["prediction_id"],
"reference_id": match["reference_id"],
"relative_area_error": match["relative_area_error"],
},
)
)
return evaluation
def _validate_rectangular_grid(
@@ -836,6 +1077,14 @@ def _validate_raster_side(
)
for index, value in enumerate(transform)
)
a, b, _x_offset, d, e, _y_offset = normalized_transform
determinant = a * e - b * d
linear_scale = max(abs(a), abs(b), abs(d), abs(e), 1.0)
if abs(determinant) <= 1e-12 * linear_scale * linear_scale:
raise ValueError(
f"{case['sample_id']}: raster affine transform is singular or "
"numerically invalid"
)
shape = context["shape"]
if (
not isinstance(shape, list)
@@ -1195,9 +1444,22 @@ def evaluate_validation(case: dict[str, Any]) -> dict[str, Any]:
observed_items = _normalized_anomalies(case, "observed_anomalies")
expected = {item["code"]: item for item in expected_items}
observed = {item["code"]: item for item in observed_items}
matched_codes = sorted(expected.keys() & observed.keys())
false_positive_codes = sorted(observed.keys() - expected.keys())
false_negative_codes = sorted(expected.keys() - observed.keys())
common_codes = sorted(expected.keys() & observed.keys())
matched_codes = [
code
for code in common_codes
if expected[code]["severity"] == observed[code]["severity"]
]
severity_mismatches = [
{
"anomaly": code,
"expected_severity": expected[code]["severity"],
"observed_severity": observed[code]["severity"],
"reason": "severity_mismatch",
}
for code in common_codes
if expected[code]["severity"] != observed[code]["severity"]
]
matched = [
{
"anomaly": code,
@@ -1208,13 +1470,31 @@ def evaluate_validation(case: dict[str, Any]) -> dict[str, Any]:
]
false_positives = [
{"anomaly": code, "severity": observed[code]["severity"]}
for code in false_positive_codes
for code in sorted(observed.keys() - expected.keys())
] + [
{
"anomaly": item["anomaly"],
"severity": item["observed_severity"],
"expected_severity": item["expected_severity"],
"reason": "severity_mismatch",
}
for item in severity_mismatches
]
false_negatives = [
{"anomaly": code, "severity": expected[code]["severity"]}
for code in false_negative_codes
for code in sorted(expected.keys() - observed.keys())
] + [
{
"anomaly": item["anomaly"],
"severity": item["expected_severity"],
"observed_severity": item["observed_severity"],
"reason": "severity_mismatch",
}
for item in severity_mismatches
]
metrics = count_metrics(len(matched), len(false_positives), len(false_negatives))
metrics["severity_mismatch_count"] = len(severity_mismatches)
metrics["severity_mismatches"] = severity_mismatches
metrics["blocker_or_critical_miss_count"] = sum(
item["severity"] in {"blocker", "critical"} for item in false_negatives
)
@@ -1259,6 +1539,7 @@ def result(
"input_lineage": serializable(case["lineage"]),
"reference_input_field": reference_field,
"prediction_input_field": prediction_field,
"classes": serializable(case.get("classes")),
"references": exact_references,
"predictions_pre_filter": exact_predictions,
"predictions_post_filter": post_filter,
@@ -1297,7 +1578,7 @@ def result(
def serializable(value: Any) -> Any:
if isinstance(value, dict):
return {
key: serializable(item) for key, item in value.items() if key != "geometry"
key: serializable(item) for key, item in value.items() if key != "_shape"
}
if isinstance(value, list):
return [serializable(item) for item in value]
@@ -1309,34 +1590,79 @@ def serializable(value: Any) -> Any:
def failure_entry(
case: dict[str, Any], kind: str, evidence: dict[str, Any]
) -> dict[str, Any]:
if case["task"] == "change_detection":
kind = (
"event_false_positive"
if kind == "false_positive"
else "event_false_negative"
)
elif case["task"] == "geospatial_data_validation":
kind = (
"validation_false_positive"
if kind == "false_positive"
else "validation_false_negative"
)
elif case["task"] == "terrain_interpretation":
kind = "terrain_missing"
base_kind = kind
if base_kind in {"false_positive", "false_negative"}:
if case["task"] == "change_detection":
kind = (
"event_false_positive"
if base_kind == "false_positive"
else "event_false_negative"
)
elif case["task"] == "geospatial_data_validation":
kind = (
"validation_false_positive"
if base_kind == "false_positive"
else "validation_false_negative"
)
elif case["task"] == "terrain_interpretation":
kind = "terrain_missing"
elif case["task"] == "raster_classification":
kind = (
"raster_misclassification"
if evidence.get("reason") == "class_mismatch"
else "raster_missing"
)
contexts: list[str] = []
secondary_error_codes: list[str] = []
metadata = case.get("metadata", {})
if metadata.get("tile_edge") is True:
contexts.append("tile_edge")
if metadata.get("ood") is True:
contexts.append("out_of_distribution")
secondary_error_codes.append("M-OOD")
confidence = evidence.get("confidence")
diagnostic_thresholds = case.get("config", {}).get(
"fixed_diagnostic_risk_thresholds"
)
valid_thresholds = (
[
float(value)
for value in diagnostic_thresholds
if isinstance(value, (int, float))
and not isinstance(value, bool)
and math.isfinite(float(value))
and 0 <= float(value) <= 1
]
if isinstance(diagnostic_thresholds, list)
else []
)
if (
base_kind == "false_positive"
and isinstance(confidence, (int, float))
and not isinstance(confidence, bool)
and valid_thresholds
and float(confidence) >= max(valid_thresholds)
):
contexts.append("high_confidence")
secondary_error_codes.append("M-MISCALIBRATED")
normalized_evidence = serializable(evidence)
return {
"failure_id": canonical_hash(
{
"sample_id": case["sample_id"],
"kind": kind,
"evidence": serializable(evidence),
"evidence": normalized_evidence,
}
)[:20],
"sample_id": case["sample_id"],
"task": case["task"],
"error_code": ERROR_CODES[kind],
"secondary_error_codes": sorted(set(secondary_error_codes)),
"contexts": sorted(set(contexts)),
"kind": kind,
"metadata": case["metadata"],
"evidence": serializable(evidence),
"metadata": serializable(metadata),
"evidence": normalized_evidence,
}
@@ -1627,6 +1953,127 @@ def _aggregate_count_family(
}
def _pooled_detection_inputs(
values: list[dict[str, Any]],
) -> tuple[
list[dict[str, Any]],
list[dict[str, Any]],
list[dict[str, Any]],
list[dict[str, Any]],
list[str | int],
]:
references: list[dict[str, Any]] = []
predictions_pre_filter: list[dict[str, Any]] = []
predictions_post_filter: list[dict[str, Any]] = []
matches: list[dict[str, Any]] = []
classes_by_hash: dict[str, str | int] = {}
for value in sorted(values, key=lambda item: item.get("sample_id", "")):
sample_id = value.get("sample_id")
if not isinstance(sample_id, str) or not sample_id:
raise ValueError("Pooled detection metrics require a sample_id")
raw = _nonempty_mapping(
value.get("raw"), f"{sample_id}: pooled detection raw evidence"
)
required_lists = (
"references",
"predictions_pre_filter",
"predictions_post_filter",
"matches",
"classes",
)
for field in required_lists:
if not isinstance(raw.get(field), list):
raise ValueError(
f"{sample_id}: pooled detection raw.{field} must be a list"
)
for class_value in raw["classes"]:
classes_by_hash[canonical_hash(class_value)] = class_value
def scoped(item: dict[str, Any], label: str) -> dict[str, Any]:
if not isinstance(item, dict):
raise ValueError(f"{sample_id}: pooled {label} must contain objects")
identifier = _stable_id(item, f"{sample_id}: pooled {label}")
return {
**item,
"id": f"{sample_id}::{identifier}",
"_sample_id": sample_id,
}
references.extend(scoped(item, "references") for item in raw["references"])
predictions_pre_filter.extend(
scoped(item, "predictions_pre_filter")
for item in raw["predictions_pre_filter"]
)
predictions_post_filter.extend(
scoped(item, "predictions_post_filter")
for item in raw["predictions_post_filter"]
)
for match in raw["matches"]:
if not isinstance(match, dict):
raise ValueError(f"{sample_id}: pooled matches must contain objects")
prediction_id = match.get("prediction_id")
if not isinstance(prediction_id, str) or not prediction_id:
raise ValueError(
f"{sample_id}: pooled match prediction_id must be non-empty"
)
matches.append({**match, "prediction_id": f"{sample_id}::{prediction_id}"})
return (
references,
predictions_pre_filter,
predictions_post_filter,
matches,
[classes_by_hash[key] for key in sorted(classes_by_hash)],
)
def _aggregate_detection(values: list[dict[str, Any]]) -> dict[str, Any]:
aggregation = _aggregate_count_family(values)
references, predictions, retained, matches, classes = _pooled_detection_inputs(
values
)
per_class = {}
for class_value in classes:
class_references = [item for item in references if item["class"] == class_value]
class_predictions = [
item for item in predictions if item["class"] == class_value
]
ap50 = detection_ap(class_predictions, class_references, 0.5)
ap50_95 = _mean_or_none(
detection_ap(class_predictions, class_references, 0.5 + step * 0.05)
for step in range(10)
)
per_class[str(class_value)] = {
"reference_count": len(class_references),
"prediction_count": len(class_predictions),
"ap50": ap50,
"ap50_95": ap50_95,
}
aggregation["micro"].update(
{
"ap50": detection_ap(predictions, references, 0.5),
"ap50_95": _mean_or_none(
detection_ap(predictions, references, 0.5 + step * 0.05)
for step in range(10)
),
"map50": _mean_or_none(item["ap50"] for item in per_class.values()),
"map50_95": _mean_or_none(item["ap50_95"] for item in per_class.values()),
"per_class_ap": per_class,
"calibration": calibration_metrics(retained, matches),
"ranking_scope": (
"predictions pooled across cases; class- and sample-aware matching; "
"no averaging of per-case AP"
),
}
)
aggregation["observation_support"].update(
{
"ranking_predictions": len(predictions),
"calibration_predictions": len(retained),
}
)
return aggregation
def _aggregate_raster(values: list[dict[str, Any]]) -> dict[str, Any]:
metrics = [item["metrics"] for item in values]
class_counts: dict[str, dict[str, int]] = defaultdict(
@@ -1727,13 +2174,14 @@ def _aggregate_terrain(values: list[dict[str, Any]]) -> dict[str, Any]:
def _aggregate_task(values: list[dict[str, Any]]) -> dict[str, Any]:
task = values[0]["task"]
if task == "raster_classification":
if task == "object_detection":
aggregation = _aggregate_detection(values)
elif task == "raster_classification":
aggregation = _aggregate_raster(values)
elif task == "terrain_interpretation":
aggregation = _aggregate_terrain(values)
else:
extras = {
"object_detection": ("ap50", "ap50_95"),
"footprint_segmentation": (
"mean_iou",
"mean_dice",
@@ -1827,6 +2275,7 @@ def subgroup_report(results: list[dict[str, Any]]) -> dict[str, Any]:
"vegetation",
"occlusion",
"difficulty",
"context",
)
dimension_reports = {}
any_insufficient = False
@@ -1881,12 +2330,32 @@ def evaluate_cases(path: Path, allowed_sample_ids: set[str]) -> dict[str, Any]:
portfolio = json.loads(path.read_text(encoding="utf-8"))
if not isinstance(portfolio, dict):
raise ValueError("The evaluation portfolio must be a JSON object")
claim_boundary = portfolio.get("claim_boundary")
if not isinstance(claim_boundary, str) or (
"synthetic" not in claim_boundary.lower()
):
schema_version = portfolio.get("schema_version")
if type(schema_version) is not int or schema_version != REPORT_SCHEMA_VERSION:
raise ValueError(
"The portfolio claim boundary must explicitly state that it is synthetic"
f"schema_version must be exactly integer {REPORT_SCHEMA_VERSION}"
)
_meaningful_string(portfolio.get("portfolio_id"), "portfolio_id")
protected_policy = portfolio.get("protected_policy")
if protected_policy != EXPECTED_PROTECTED_POLICY:
raise ValueError(
"protected_policy must exactly equal the frozen no-selection policy: "
f"{EXPECTED_PROTECTED_POLICY}"
)
portfolio_kind = portfolio.get("portfolio_kind")
if portfolio_kind not in PORTFOLIO_KINDS:
raise ValueError(f"portfolio_kind must be one of {sorted(PORTFOLIO_KINDS)}")
claim_boundary = portfolio.get("claim_boundary")
claim_lower = claim_boundary.lower() if isinstance(claim_boundary, str) else ""
if portfolio_kind == "synthetic_contract":
if "synthetic" not in claim_lower:
raise ValueError(
"A synthetic_contract claim boundary must explicitly state that it is synthetic"
)
elif "governed product baseline" not in claim_lower or "synthetic" in claim_lower:
raise ValueError(
"A governed_product_baseline claim boundary must explicitly state "
"'governed product baseline' and must not describe the portfolio as synthetic"
)
split_roles = portfolio.get("split_roles")
if split_roles not in (["test"], ["test", "background-test"]):
@@ -1895,17 +2364,18 @@ def evaluate_cases(path: Path, allowed_sample_ids: set[str]) -> dict[str, Any]:
)
if "challenge_cases" in portfolio or "challenge_labels" in portfolio:
raise ValueError("Challenge cases and labels must remain sealed and absent")
selection_policy = portfolio.get("selection_policy")
if not isinstance(selection_policy, str) or not selection_policy.strip():
raise ValueError("The portfolio requires an explicit selection policy")
selection_policy = _meaningful_string(
portfolio.get("selection_policy"), "selection_policy"
)
declared_portfolio_lineage = _nonempty_mapping(
portfolio.get("portfolio_lineage"),
"portfolio_lineage",
)
for field in ("origin", "source_path", "version"):
value = declared_portfolio_lineage.get(field)
if not isinstance(value, str) or not value.strip():
raise ValueError(f"portfolio_lineage.{field} must be a non-empty string")
_meaningful_string(
declared_portfolio_lineage.get(field),
f"portfolio_lineage.{field}",
)
cases = portfolio.get("cases")
if not isinstance(cases, list) or not all(isinstance(item, dict) for item in cases):
raise ValueError("The portfolio cases must be an object list")
@@ -1923,6 +2393,13 @@ def evaluate_cases(path: Path, allowed_sample_ids: set[str]) -> dict[str, Any]:
if duplicates:
raise ValueError(f"Duplicate protected case ids: {duplicates}")
case_ids = set(identifiers)
observed_tasks = {item["task"] for item in cases}
if portfolio_kind == "governed_product_baseline" and observed_tasks != TASKS:
raise ValueError(
"A governed_product_baseline must cover every evaluator task family: "
f"missing={sorted(TASKS - observed_tasks)}, "
f"unexpected={sorted(observed_tasks - TASKS)}"
)
unexpected = sorted(case_ids - allowed_sample_ids)
missing = sorted(allowed_sample_ids - case_ids)
if unexpected or missing:
@@ -1939,6 +2416,7 @@ def evaluate_cases(path: Path, allowed_sample_ids: set[str]) -> dict[str, Any]:
"declared": serializable(declared_portfolio_lineage),
"portfolio_id": portfolio.get("portfolio_id"),
"portfolio_schema_version": portfolio.get("schema_version"),
"portfolio_kind": portfolio_kind,
"portfolio_file_sha256": file_hash,
"portfolio_canonical_json_sha256": portfolio_canonical_hash,
"source_path": declared_portfolio_lineage["source_path"],
@@ -1947,6 +2425,13 @@ def evaluate_cases(path: Path, allowed_sample_ids: set[str]) -> dict[str, Any]:
}
for item in results:
item["raw"]["portfolio_lineage"] = portfolio_lineage
results_by_task: dict[str, list[dict[str, Any]]] = defaultdict(list)
for item in results:
results_by_task[item["task"]].append(item)
portfolio_metrics = {
task: _aggregate_task(task_results)
for task, task_results in sorted(results_by_task.items())
}
failures = sorted(
(failure for item in results for failure in item["failures"]),
key=lambda item: item["failure_id"],
@@ -1956,6 +2441,7 @@ def evaluate_cases(path: Path, allowed_sample_ids: set[str]) -> dict[str, Any]:
"schema_version": REPORT_SCHEMA_VERSION,
"evaluator_version": EVALUATOR_VERSION,
"portfolio_id": portfolio["portfolio_id"],
"portfolio_kind": portfolio_kind,
"portfolio_file_sha256": file_hash,
"portfolio_canonical_json_sha256": portfolio_canonical_hash,
"claim_boundary": claim_boundary,
@@ -1978,5 +2464,11 @@ def evaluate_cases(path: Path, allowed_sample_ids: set[str]) -> dict[str, Any]:
"results": results,
"subgroups": subgroup_report(results),
"failures": failures,
"failure_taxonomy": {
"primary_codes": FAILURE_TAXONOMY,
"contexts": FAILURE_CONTEXTS,
"claim_boundary": "diagnostic classification; not a release decision",
},
"portfolio_metrics": portfolio_metrics,
"results_canonical_json_sha256": results_hash,
}
scripts/generate_accuracy_phase4_splits.py
+1341 -76
View File
File diff suppressed because it is too large Load Diff
scripts/run_accuracy_phase4_benchmark.py
+2263 -195
View File
File diff suppressed because it is too large Load Diff