Jens/geointel
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(accuracy): close phase 4 evidence bypasses

Browse Source
This commit is contained in:
Jens
2026-08-02 04:29:10 +02:00
parent bd2fd9780f
commit ee1982eff2
7 changed files with 5814 additions and 382 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/accuracy_phase4_evaluator.py
+567 -75
View File
@@ -1,7 +1,8 @@
"""Fail-closed, task-aware metrics for the GeoIntel Phase 4 benchmark.
The bundled portfolio is a synthetic contract fixture. Nothing emitted by
this module is, by itself, evidence of production-model accuracy.
The evaluator accepts explicitly separated synthetic contract portfolios and
governed product-baseline portfolios. Nothing emitted by this module is, by
itself, evidence that inference provenance or production release gates passed.
"""
from __future__ import annotations
@@ -14,7 +15,8 @@ from pathlib import Path
from statistics import mean, median
from typing import Any, Callable, Iterable
EVALUATOR_VERSION = "2.0.0"
PORTFOLIO_KINDS = {"synthetic_contract", "governed_product_baseline"}
EVALUATOR_VERSION = "2.1.0"
REPORT_SCHEMA_VERSION = 2
SUBGROUP_MIN_CASE_SUPPORT = 5
CANONICAL_JSON_SPEC = (
@@ -46,6 +48,54 @@ ERROR_CODES = {
"validation_false_positive": "D-VALIDATION-FP",
"validation_false_negative": "D-VALIDATION-FN",
"terrain_missing": "P-PARTIAL",
"raster_misclassification": "M-CLASS",
"raster_missing": "P-PARTIAL",
"boundary_error": "M-BOUNDARY",
"area_bias": "M-AREA-BIAS",
"miscalibrated": "M-MISCALIBRATED",
}
EXPECTED_PROTECTED_POLICY = {
"operating_point_selection_allowed": False,
"diagnostic_curves_select_operating_point": False,
"test_feedback_allowed": False,
"threshold_selection_source": "pre_registered_configuration_only",
}
REQUIRED_METADATA_STRING_FIELDS = (
"region",
"municipality",
"urbanity",
"object_size",
"source",
"sensor",
"season",
"date",
"vegetation",
"occlusion",
"difficulty",
"context",
)
LINEAGE_SIDES = ("reference", "prediction")
LINEAGE_FIELDS = ("source_id", "source_version", "derivation")
NON_MEANINGFUL_TOKENS = {"", "unknown", "n/a", "na", "null", "tbd", "todo"}
BELGIUM_SCOPE_BOUNDS = (1.9, 49.4, 7.5, 52.1)
FAILURE_TAXONOMY = {
"M-FP-CONFUSER": "unmatched object or event prediction",
"M-FN-MISSED": "unmatched reference object or event",
"M-CLASS": "raster class differs from the valid reference class",
"M-BOUNDARY": "matched footprint has a measurable boundary deviation",
"M-AREA-BIAS": "matched footprint has a measurable relative area bias",
"M-MISCALIBRATED": "confidence diagnostic deviates from observed correctness",
"M-OOD": "error observed in an explicitly declared out-of-distribution case",
"D-VALIDATION-FP": "anomaly or severity reported without an exact reference match",
"D-VALIDATION-FN": "reference anomaly or severity not exactly reported",
"P-PARTIAL": "required prediction value is unavailable",
}
FAILURE_CONTEXTS = {
"tile_edge": "case metadata explicitly marks tile-edge context",
"high_confidence": (
"false positive meets the highest frozen diagnostic confidence threshold"
),
"out_of_distribution": "case metadata explicitly marks OOD context",
}
@@ -146,17 +196,62 @@ def _nonempty_mapping(value: Any, label: str) -> dict[str, Any]:
return value
def _meaningful_string(
value: Any,
label: str,
*,
allow_not_applicable: bool = False,
) -> str:
if not isinstance(value, str) or value.strip().lower() in NON_MEANINGFUL_TOKENS:
raise ValueError(f"{label} must be a meaningful non-empty string")
normalized = value.strip()
if not allow_not_applicable and normalized.lower() == "not_applicable":
raise ValueError(f"{label} may not be not_applicable")
return normalized
def _validate_lineage(case: dict[str, Any]) -> None:
sample_id = case["sample_id"]
lineage = _nonempty_mapping(case.get("lineage"), f"{sample_id}: lineage")
missing_sides = sorted(set(LINEAGE_SIDES) - lineage.keys())
if missing_sides:
raise ValueError(f"{sample_id}: lineage missing {missing_sides}")
for side in LINEAGE_SIDES:
entry = _nonempty_mapping(lineage.get(side), f"{sample_id}: lineage.{side}")
missing_fields = sorted(set(LINEAGE_FIELDS) - entry.keys())
if missing_fields:
raise ValueError(f"{sample_id}: lineage.{side} missing {missing_fields}")
for field in LINEAGE_FIELDS:
_meaningful_string(
entry.get(field),
f"{sample_id}: lineage.{side}.{field}",
)
def _validate_case_common(case: dict[str, Any]) -> None:
sample_id = case.get("sample_id")
if not isinstance(sample_id, str) or not sample_id.strip():
raise ValueError("Every case requires a non-empty sample_id")
if case.get("task") not in TASKS:
raise ValueError(f"{sample_id}: unsupported task {case.get('task')!r}")
if not isinstance(case.get("metadata"), dict):
raise ValueError(f"{sample_id}: metadata must be an object")
metadata = _nonempty_mapping(case.get("metadata"), f"{sample_id}: metadata")
missing_metadata = sorted(set(REQUIRED_METADATA_STRING_FIELDS) - metadata.keys())
if missing_metadata:
raise ValueError(f"{sample_id}: metadata missing {missing_metadata}")
for field in REQUIRED_METADATA_STRING_FIELDS:
_meaningful_string(
metadata.get(field),
f"{sample_id}: metadata.{field}",
allow_not_applicable=True,
)
resolution = metadata.get("resolution_m")
if case["task"] == "geospatial_data_validation" and resolution is None:
pass
elif _finite_float(resolution, f"{sample_id}: metadata.resolution_m") <= 0:
raise ValueError(f"{sample_id}: metadata.resolution_m must be positive")
if not isinstance(case.get("config"), dict):
raise ValueError(f"{sample_id}: config must be an explicit object")
_nonempty_mapping(case.get("lineage"), f"{sample_id}: lineage")
_validate_lineage(case)
def _stable_id(item: dict[str, Any], label: str) -> str:
@@ -339,6 +434,8 @@ def detection_ap(
reference = references[index]
if not _class_compatible(prediction, reference):
continue
if prediction.get("_sample_id") != reference.get("_sample_id"):
continue
candidates.append(
(
bbox_iou(prediction["bbox"], reference["bbox"]),
@@ -510,6 +607,18 @@ def evaluate_object_detection(case: dict[str, Any]) -> dict[str, Any]:
case["config"].get("match_iou"),
f"{case['sample_id']}.config.match_iou",
)
diagnostic_thresholds = case["config"].get("fixed_diagnostic_risk_thresholds")
if diagnostic_thresholds is not None:
if not isinstance(diagnostic_thresholds, list) or not diagnostic_thresholds:
raise ValueError(
f"{case['sample_id']}: fixed diagnostic risk thresholds must be a "
"non-empty list"
)
for index, value in enumerate(diagnostic_thresholds):
_probability(
value,
f"{case['sample_id']}.config.fixed_diagnostic_risk_thresholds[{index}]",
)
retained = [item for item in predictions if float(item["confidence"]) >= threshold]
matches, false_positives, false_negatives = greedy_match(
retained,
@@ -538,6 +647,10 @@ def evaluate_object_detection(case: dict[str, Any]) -> dict[str, Any]:
class_metrics["ap50"] = detection_ap(
all_class_predictions, class_references, 0.5
)
class_metrics["ap50_95"] = _mean_or_none(
detection_ap(all_class_predictions, class_references, 0.5 + step * 0.05)
for step in range(10)
)
per_class[str(class_value)] = class_metrics
metrics = count_metrics(len(matches), len(false_positives), len(false_negatives))
metrics.update(
@@ -555,9 +668,11 @@ def evaluate_object_detection(case: dict[str, Any]) -> dict[str, Any]:
"coverage_risk": coverage_risk(
predictions, references, match_iou, threshold
),
"map50": _mean_or_none(item["ap50"] for item in per_class.values()),
"map50_95": _mean_or_none(item["ap50_95"] for item in per_class.values()),
}
)
return result(
evaluation = result(
case,
metrics,
matches,
@@ -571,12 +686,24 @@ def evaluate_object_detection(case: dict[str, Any]) -> dict[str, Any]:
"threshold": threshold,
},
)
def polygon(item: dict[str, Any]):
from shapely.geometry import Polygon
return Polygon(item["polygon"])
calibration = metrics["calibration"]
if calibration["status"] == "computed" and calibration["ece"] > 1e-12:
evaluation["failures"].append(
failure_entry(
case,
"miscalibrated",
{
"ece": calibration["ece"],
"brier": calibration["brier"],
"prediction_support": len(retained),
"claim_boundary": (
"diagnostic deviation only; not a representative "
"population-calibration claim"
),
},
)
)
return evaluation
def distribution(values: list[float]) -> dict[str, Any]:
@@ -594,6 +721,18 @@ def distribution(values: list[float]) -> dict[str, Any]:
}
def _bounds_overlap(
left: tuple[float, float, float, float],
right: tuple[float, float, float, float],
) -> bool:
return not (
left[2] < right[0]
or left[0] > right[2]
or left[3] < right[1]
or left[1] > right[3]
)
def _validate_metric_spatial_context(case: dict[str, Any]) -> dict[str, Any]:
context = _nonempty_mapping(
case.get("spatial_context"),
@@ -629,32 +768,99 @@ def _validate_metric_spatial_context(case: dict[str, Any]) -> dict[str, Any]:
for axis in crs.axis_info[:2]
):
raise ValueError(f"{case['sample_id']}: CRS axes must use metres")
if crs.to_epsg() in {3395, 3857}:
raise ValueError(
f"{case['sample_id']}: Web/World Mercator is unsuitable for "
"benchmark area, boundary and distance metrics"
)
area = crs.area_of_use
if area is None:
raise ValueError(
f"{case['sample_id']}: CRS area of use is unavailable; Belgian "
"metric suitability cannot be proven"
)
crs_bounds = (area.west, area.south, area.east, area.north)
if not _bounds_overlap(crs_bounds, BELGIUM_SCOPE_BOUNDS):
raise ValueError(
f"{case['sample_id']}: CRS area of use does not overlap the "
"declared Belgium and Belgian North Sea product scope"
)
return context
def _validated_polygon(item: dict[str, Any], label: str):
coordinates = item.get("polygon")
def _validated_ring(coordinates: Any, label: str) -> list[list[float]]:
if not isinstance(coordinates, list) or len(coordinates) < 4:
raise ValueError(
f"{label}.polygon must contain a closed ring with at least four points"
f"{label} must contain a closed ring with at least four points"
)
normalized: list[tuple[float, float]] = []
normalized: list[list[float]] = []
for index, point in enumerate(coordinates):
if not isinstance(point, list) or len(point) != 2:
raise ValueError(f"{label}.polygon[{index}] must be an [x, y] pair")
raise ValueError(f"{label}[{index}] must be an [x, y] pair")
normalized.append(
(
_finite_float(point[0], f"{label}.polygon[{index}][0]"),
_finite_float(point[1], f"{label}.polygon[{index}][1]"),
)
[
_finite_float(point[0], f"{label}[{index}][0]"),
_finite_float(point[1], f"{label}[{index}][1]"),
]
)
if normalized[0] != normalized[-1]:
raise ValueError(f"{label}.polygon ring must be explicitly closed")
from shapely.geometry import Polygon
raise ValueError(f"{label} must be explicitly closed")
return normalized
geometry = Polygon(normalized)
def _validated_polygon(item: dict[str, Any], label: str):
if "geometry" in item and "polygon" in item:
raise ValueError(f"{label} may not declare both geometry and polygon")
if "geometry" in item:
payload = item["geometry"]
if not isinstance(payload, dict):
raise ValueError(f"{label}.geometry must be a GeoJSON object")
geometry_type = payload.get("type")
coordinates = payload.get("coordinates")
if geometry_type == "Polygon":
if not isinstance(coordinates, list) or not coordinates:
raise ValueError(f"{label}.geometry Polygon requires rings")
normalized_coordinates: Any = [
_validated_ring(ring, f"{label}.geometry.coordinates[{index}]")
for index, ring in enumerate(coordinates)
]
elif geometry_type == "MultiPolygon":
if not isinstance(coordinates, list) or not coordinates:
raise ValueError(
f"{label}.geometry MultiPolygon requires polygon members"
)
normalized_coordinates = []
for polygon_index, polygon_coordinates in enumerate(coordinates):
if not isinstance(polygon_coordinates, list) or not polygon_coordinates:
raise ValueError(
f"{label}.geometry.coordinates[{polygon_index}] requires rings"
)
normalized_coordinates.append(
[
_validated_ring(
ring,
f"{label}.geometry.coordinates[{polygon_index}]"
f"[{ring_index}]",
)
for ring_index, ring in enumerate(polygon_coordinates)
]
)
else:
raise ValueError(f"{label}.geometry.type must be Polygon or MultiPolygon")
normalized_payload = {
"type": geometry_type,
"coordinates": normalized_coordinates,
}
else:
normalized_payload = {
"type": "Polygon",
"coordinates": [_validated_ring(item.get("polygon"), f"{label}.polygon")],
}
from shapely.geometry import shape
geometry = shape(normalized_payload)
if geometry.is_empty or geometry.area <= 0 or not geometry.is_valid:
raise ValueError(f"{label}.polygon must be non-empty, positive-area and valid")
raise ValueError(f"{label}.geometry must be non-empty, positive-area and valid")
return geometry
@@ -679,16 +885,14 @@ def _validated_polygon_items(
validated_references = [
dict(
item,
geometry=_validated_polygon(
item, f"{case['sample_id']}.references[{index}]"
),
_shape=_validated_polygon(item, f"{case['sample_id']}.references[{index}]"),
)
for index, item in enumerate(references)
]
validated_predictions = [
dict(
item,
geometry=_validated_polygon(
_shape=_validated_polygon(
item, f"{case['sample_id']}.predictions[{index}]"
),
)
@@ -698,8 +902,8 @@ def _validated_polygon_items(
def _polygon_overlap(prediction: dict[str, Any], reference: dict[str, Any]) -> float:
intersection = prediction["geometry"].intersection(reference["geometry"]).area
union = prediction["geometry"].union(reference["geometry"]).area
intersection = prediction["_shape"].intersection(reference["_shape"]).area
union = prediction["_shape"].union(reference["_shape"]).area
return intersection / union if union > 0 else 0.0
@@ -753,13 +957,23 @@ def evaluate_footprint_segmentation(case: dict[str, Any]) -> dict[str, Any]:
centroids = []
area_errors = []
for match in matches:
pred = by_prediction[match["prediction_id"]]["geometry"]
ref = by_reference[match["reference_id"]]["geometry"]
pred = by_prediction[match["prediction_id"]]["_shape"]
ref = by_reference[match["reference_id"]]["_shape"]
intersection = pred.intersection(ref).area
dice.append(2 * intersection / (pred.area + ref.area))
boundary.append(boundary_f1(pred, ref, tolerance))
centroids.append(pred.centroid.distance(ref.centroid))
area_errors.append((pred.area - ref.area) / ref.area)
dice_value = 2 * intersection / (pred.area + ref.area)
boundary_value = boundary_f1(pred, ref, tolerance)
centroid_value = pred.centroid.distance(ref.centroid)
area_error = (pred.area - ref.area) / ref.area
dice.append(dice_value)
boundary.append(boundary_value)
centroids.append(centroid_value)
area_errors.append(area_error)
match.update(
dice=dice_value,
boundary_f1=boundary_value,
centroid_distance_m=centroid_value,
relative_area_error=area_error,
)
metrics = count_metrics(len(matches), len(false_positives), len(false_negatives))
metrics.update(
{
@@ -773,13 +987,40 @@ def evaluate_footprint_segmentation(case: dict[str, Any]) -> dict[str, Any]:
"spatial_context_validated": True,
}
)
return result(
evaluation = result(
case,
metrics,
matches,
false_positives,
false_negatives,
)
for match in matches:
if match["boundary_f1"] is not None and match["boundary_f1"] < 1 - 1e-12:
evaluation["failures"].append(
failure_entry(
case,
"boundary_error",
{
"prediction_id": match["prediction_id"],
"reference_id": match["reference_id"],
"boundary_f1": match["boundary_f1"],
"boundary_tolerance_m": tolerance,
},
)
)
if abs(match["relative_area_error"]) > 1e-12:
evaluation["failures"].append(
failure_entry(
case,
"area_bias",
{
"prediction_id": match["prediction_id"],
"reference_id": match["reference_id"],
"relative_area_error": match["relative_area_error"],
},
)
)
return evaluation
def _validate_rectangular_grid(
@@ -836,6 +1077,14 @@ def _validate_raster_side(
)
for index, value in enumerate(transform)
)
a, b, _x_offset, d, e, _y_offset = normalized_transform
determinant = a * e - b * d
linear_scale = max(abs(a), abs(b), abs(d), abs(e), 1.0)
if abs(determinant) <= 1e-12 * linear_scale * linear_scale:
raise ValueError(
f"{case['sample_id']}: raster affine transform is singular or "
"numerically invalid"
)
shape = context["shape"]
if (
not isinstance(shape, list)
@@ -1195,9 +1444,22 @@ def evaluate_validation(case: dict[str, Any]) -> dict[str, Any]:
observed_items = _normalized_anomalies(case, "observed_anomalies")
expected = {item["code"]: item for item in expected_items}
observed = {item["code"]: item for item in observed_items}
matched_codes = sorted(expected.keys() & observed.keys())
false_positive_codes = sorted(observed.keys() - expected.keys())
false_negative_codes = sorted(expected.keys() - observed.keys())
common_codes = sorted(expected.keys() & observed.keys())
matched_codes = [
code
for code in common_codes
if expected[code]["severity"] == observed[code]["severity"]
]
severity_mismatches = [
{
"anomaly": code,
"expected_severity": expected[code]["severity"],
"observed_severity": observed[code]["severity"],
"reason": "severity_mismatch",
}
for code in common_codes
if expected[code]["severity"] != observed[code]["severity"]
]
matched = [
{
"anomaly": code,
@@ -1208,13 +1470,31 @@ def evaluate_validation(case: dict[str, Any]) -> dict[str, Any]:
]
false_positives = [
{"anomaly": code, "severity": observed[code]["severity"]}
for code in false_positive_codes
for code in sorted(observed.keys() - expected.keys())
] + [
{
"anomaly": item["anomaly"],
"severity": item["observed_severity"],
"expected_severity": item["expected_severity"],
"reason": "severity_mismatch",
}
for item in severity_mismatches
]
false_negatives = [
{"anomaly": code, "severity": expected[code]["severity"]}
for code in false_negative_codes
for code in sorted(expected.keys() - observed.keys())
] + [
{
"anomaly": item["anomaly"],
"severity": item["expected_severity"],
"observed_severity": item["observed_severity"],
"reason": "severity_mismatch",
}
for item in severity_mismatches
]
metrics = count_metrics(len(matched), len(false_positives), len(false_negatives))
metrics["severity_mismatch_count"] = len(severity_mismatches)
metrics["severity_mismatches"] = severity_mismatches
metrics["blocker_or_critical_miss_count"] = sum(
item["severity"] in {"blocker", "critical"} for item in false_negatives
)
@@ -1259,6 +1539,7 @@ def result(
"input_lineage": serializable(case["lineage"]),
"reference_input_field": reference_field,
"prediction_input_field": prediction_field,
"classes": serializable(case.get("classes")),
"references": exact_references,
"predictions_pre_filter": exact_predictions,
"predictions_post_filter": post_filter,
@@ -1297,7 +1578,7 @@ def result(
def serializable(value: Any) -> Any:
if isinstance(value, dict):
return {
key: serializable(item) for key, item in value.items() if key != "geometry"
key: serializable(item) for key, item in value.items() if key != "_shape"
}
if isinstance(value, list):
return [serializable(item) for item in value]
@@ -1309,34 +1590,79 @@ def serializable(value: Any) -> Any:
def failure_entry(
case: dict[str, Any], kind: str, evidence: dict[str, Any]
) -> dict[str, Any]:
if case["task"] == "change_detection":
kind = (
"event_false_positive"
if kind == "false_positive"
else "event_false_negative"
)
elif case["task"] == "geospatial_data_validation":
kind = (
"validation_false_positive"
if kind == "false_positive"
else "validation_false_negative"
)
elif case["task"] == "terrain_interpretation":
kind = "terrain_missing"
base_kind = kind
if base_kind in {"false_positive", "false_negative"}:
if case["task"] == "change_detection":
kind = (
"event_false_positive"
if base_kind == "false_positive"
else "event_false_negative"
)
elif case["task"] == "geospatial_data_validation":
kind = (
"validation_false_positive"
if base_kind == "false_positive"
else "validation_false_negative"
)
elif case["task"] == "terrain_interpretation":
kind = "terrain_missing"
elif case["task"] == "raster_classification":
kind = (
"raster_misclassification"
if evidence.get("reason") == "class_mismatch"
else "raster_missing"
)
contexts: list[str] = []
secondary_error_codes: list[str] = []
metadata = case.get("metadata", {})
if metadata.get("tile_edge") is True:
contexts.append("tile_edge")
if metadata.get("ood") is True:
contexts.append("out_of_distribution")
secondary_error_codes.append("M-OOD")
confidence = evidence.get("confidence")
diagnostic_thresholds = case.get("config", {}).get(
"fixed_diagnostic_risk_thresholds"
)
valid_thresholds = (
[
float(value)
for value in diagnostic_thresholds
if isinstance(value, (int, float))
and not isinstance(value, bool)
and math.isfinite(float(value))
and 0 <= float(value) <= 1
]
if isinstance(diagnostic_thresholds, list)
else []
)
if (
base_kind == "false_positive"
and isinstance(confidence, (int, float))
and not isinstance(confidence, bool)
and valid_thresholds
and float(confidence) >= max(valid_thresholds)
):
contexts.append("high_confidence")
secondary_error_codes.append("M-MISCALIBRATED")
normalized_evidence = serializable(evidence)
return {
"failure_id": canonical_hash(
{
"sample_id": case["sample_id"],
"kind": kind,
"evidence": serializable(evidence),
"evidence": normalized_evidence,
}
)[:20],
"sample_id": case["sample_id"],
"task": case["task"],
"error_code": ERROR_CODES[kind],
"secondary_error_codes": sorted(set(secondary_error_codes)),
"contexts": sorted(set(contexts)),
"kind": kind,
"metadata": case["metadata"],
"evidence": serializable(evidence),
"metadata": serializable(metadata),
"evidence": normalized_evidence,
}
@@ -1627,6 +1953,127 @@ def _aggregate_count_family(
}
def _pooled_detection_inputs(
values: list[dict[str, Any]],
) -> tuple[
list[dict[str, Any]],
list[dict[str, Any]],
list[dict[str, Any]],
list[dict[str, Any]],
list[str | int],
]:
references: list[dict[str, Any]] = []
predictions_pre_filter: list[dict[str, Any]] = []
predictions_post_filter: list[dict[str, Any]] = []
matches: list[dict[str, Any]] = []
classes_by_hash: dict[str, str | int] = {}
for value in sorted(values, key=lambda item: item.get("sample_id", "")):
sample_id = value.get("sample_id")
if not isinstance(sample_id, str) or not sample_id:
raise ValueError("Pooled detection metrics require a sample_id")
raw = _nonempty_mapping(
value.get("raw"), f"{sample_id}: pooled detection raw evidence"
)
required_lists = (
"references",
"predictions_pre_filter",
"predictions_post_filter",
"matches",
"classes",
)
for field in required_lists:
if not isinstance(raw.get(field), list):
raise ValueError(
f"{sample_id}: pooled detection raw.{field} must be a list"
)
for class_value in raw["classes"]:
classes_by_hash[canonical_hash(class_value)] = class_value
def scoped(item: dict[str, Any], label: str) -> dict[str, Any]:
if not isinstance(item, dict):
raise ValueError(f"{sample_id}: pooled {label} must contain objects")
identifier = _stable_id(item, f"{sample_id}: pooled {label}")
return {
**item,
"id": f"{sample_id}::{identifier}",
"_sample_id": sample_id,
}
references.extend(scoped(item, "references") for item in raw["references"])
predictions_pre_filter.extend(
scoped(item, "predictions_pre_filter")
for item in raw["predictions_pre_filter"]
)
predictions_post_filter.extend(
scoped(item, "predictions_post_filter")
for item in raw["predictions_post_filter"]
)
for match in raw["matches"]:
if not isinstance(match, dict):
raise ValueError(f"{sample_id}: pooled matches must contain objects")
prediction_id = match.get("prediction_id")
if not isinstance(prediction_id, str) or not prediction_id:
raise ValueError(
f"{sample_id}: pooled match prediction_id must be non-empty"
)
matches.append({**match, "prediction_id": f"{sample_id}::{prediction_id}"})
return (
references,
predictions_pre_filter,
predictions_post_filter,
matches,
[classes_by_hash[key] for key in sorted(classes_by_hash)],
)
def _aggregate_detection(values: list[dict[str, Any]]) -> dict[str, Any]:
aggregation = _aggregate_count_family(values)
references, predictions, retained, matches, classes = _pooled_detection_inputs(
values
)
per_class = {}
for class_value in classes:
class_references = [item for item in references if item["class"] == class_value]
class_predictions = [
item for item in predictions if item["class"] == class_value
]
ap50 = detection_ap(class_predictions, class_references, 0.5)
ap50_95 = _mean_or_none(
detection_ap(class_predictions, class_references, 0.5 + step * 0.05)
for step in range(10)
)
per_class[str(class_value)] = {
"reference_count": len(class_references),
"prediction_count": len(class_predictions),
"ap50": ap50,
"ap50_95": ap50_95,
}
aggregation["micro"].update(
{
"ap50": detection_ap(predictions, references, 0.5),
"ap50_95": _mean_or_none(
detection_ap(predictions, references, 0.5 + step * 0.05)
for step in range(10)
),
"map50": _mean_or_none(item["ap50"] for item in per_class.values()),
"map50_95": _mean_or_none(item["ap50_95"] for item in per_class.values()),
"per_class_ap": per_class,
"calibration": calibration_metrics(retained, matches),
"ranking_scope": (
"predictions pooled across cases; class- and sample-aware matching; "
"no averaging of per-case AP"
),
}
)
aggregation["observation_support"].update(
{
"ranking_predictions": len(predictions),
"calibration_predictions": len(retained),
}
)
return aggregation
def _aggregate_raster(values: list[dict[str, Any]]) -> dict[str, Any]:
metrics = [item["metrics"] for item in values]
class_counts: dict[str, dict[str, int]] = defaultdict(
@@ -1727,13 +2174,14 @@ def _aggregate_terrain(values: list[dict[str, Any]]) -> dict[str, Any]:
def _aggregate_task(values: list[dict[str, Any]]) -> dict[str, Any]:
task = values[0]["task"]
if task == "raster_classification":
if task == "object_detection":
aggregation = _aggregate_detection(values)
elif task == "raster_classification":
aggregation = _aggregate_raster(values)
elif task == "terrain_interpretation":
aggregation = _aggregate_terrain(values)
else:
extras = {
"object_detection": ("ap50", "ap50_95"),
"footprint_segmentation": (
"mean_iou",
"mean_dice",
@@ -1827,6 +2275,7 @@ def subgroup_report(results: list[dict[str, Any]]) -> dict[str, Any]:
"vegetation",
"occlusion",
"difficulty",
"context",
)
dimension_reports = {}
any_insufficient = False
@@ -1881,12 +2330,32 @@ def evaluate_cases(path: Path, allowed_sample_ids: set[str]) -> dict[str, Any]:
portfolio = json.loads(path.read_text(encoding="utf-8"))
if not isinstance(portfolio, dict):
raise ValueError("The evaluation portfolio must be a JSON object")
claim_boundary = portfolio.get("claim_boundary")
if not isinstance(claim_boundary, str) or (
"synthetic" not in claim_boundary.lower()
):
schema_version = portfolio.get("schema_version")
if type(schema_version) is not int or schema_version != REPORT_SCHEMA_VERSION:
raise ValueError(
"The portfolio claim boundary must explicitly state that it is synthetic"
f"schema_version must be exactly integer {REPORT_SCHEMA_VERSION}"
)
_meaningful_string(portfolio.get("portfolio_id"), "portfolio_id")
protected_policy = portfolio.get("protected_policy")
if protected_policy != EXPECTED_PROTECTED_POLICY:
raise ValueError(
"protected_policy must exactly equal the frozen no-selection policy: "
f"{EXPECTED_PROTECTED_POLICY}"
)
portfolio_kind = portfolio.get("portfolio_kind")
if portfolio_kind not in PORTFOLIO_KINDS:
raise ValueError(f"portfolio_kind must be one of {sorted(PORTFOLIO_KINDS)}")
claim_boundary = portfolio.get("claim_boundary")
claim_lower = claim_boundary.lower() if isinstance(claim_boundary, str) else ""
if portfolio_kind == "synthetic_contract":
if "synthetic" not in claim_lower:
raise ValueError(
"A synthetic_contract claim boundary must explicitly state that it is synthetic"
)
elif "governed product baseline" not in claim_lower or "synthetic" in claim_lower:
raise ValueError(
"A governed_product_baseline claim boundary must explicitly state "
"'governed product baseline' and must not describe the portfolio as synthetic"
)
split_roles = portfolio.get("split_roles")
if split_roles not in (["test"], ["test", "background-test"]):
@@ -1895,17 +2364,18 @@ def evaluate_cases(path: Path, allowed_sample_ids: set[str]) -> dict[str, Any]:
)
if "challenge_cases" in portfolio or "challenge_labels" in portfolio:
raise ValueError("Challenge cases and labels must remain sealed and absent")
selection_policy = portfolio.get("selection_policy")
if not isinstance(selection_policy, str) or not selection_policy.strip():
raise ValueError("The portfolio requires an explicit selection policy")
selection_policy = _meaningful_string(
portfolio.get("selection_policy"), "selection_policy"
)
declared_portfolio_lineage = _nonempty_mapping(
portfolio.get("portfolio_lineage"),
"portfolio_lineage",
)
for field in ("origin", "source_path", "version"):
value = declared_portfolio_lineage.get(field)
if not isinstance(value, str) or not value.strip():
raise ValueError(f"portfolio_lineage.{field} must be a non-empty string")
_meaningful_string(
declared_portfolio_lineage.get(field),
f"portfolio_lineage.{field}",
)
cases = portfolio.get("cases")
if not isinstance(cases, list) or not all(isinstance(item, dict) for item in cases):
raise ValueError("The portfolio cases must be an object list")
@@ -1923,6 +2393,13 @@ def evaluate_cases(path: Path, allowed_sample_ids: set[str]) -> dict[str, Any]:
if duplicates:
raise ValueError(f"Duplicate protected case ids: {duplicates}")
case_ids = set(identifiers)
observed_tasks = {item["task"] for item in cases}
if portfolio_kind == "governed_product_baseline" and observed_tasks != TASKS:
raise ValueError(
"A governed_product_baseline must cover every evaluator task family: "
f"missing={sorted(TASKS - observed_tasks)}, "
f"unexpected={sorted(observed_tasks - TASKS)}"
)
unexpected = sorted(case_ids - allowed_sample_ids)
missing = sorted(allowed_sample_ids - case_ids)
if unexpected or missing:
@@ -1939,6 +2416,7 @@ def evaluate_cases(path: Path, allowed_sample_ids: set[str]) -> dict[str, Any]:
"declared": serializable(declared_portfolio_lineage),
"portfolio_id": portfolio.get("portfolio_id"),
"portfolio_schema_version": portfolio.get("schema_version"),
"portfolio_kind": portfolio_kind,
"portfolio_file_sha256": file_hash,
"portfolio_canonical_json_sha256": portfolio_canonical_hash,
"source_path": declared_portfolio_lineage["source_path"],
@@ -1947,6 +2425,13 @@ def evaluate_cases(path: Path, allowed_sample_ids: set[str]) -> dict[str, Any]:
}
for item in results:
item["raw"]["portfolio_lineage"] = portfolio_lineage
results_by_task: dict[str, list[dict[str, Any]]] = defaultdict(list)
for item in results:
results_by_task[item["task"]].append(item)
portfolio_metrics = {
task: _aggregate_task(task_results)
for task, task_results in sorted(results_by_task.items())
}
failures = sorted(
(failure for item in results for failure in item["failures"]),
key=lambda item: item["failure_id"],
@@ -1956,6 +2441,7 @@ def evaluate_cases(path: Path, allowed_sample_ids: set[str]) -> dict[str, Any]:
"schema_version": REPORT_SCHEMA_VERSION,
"evaluator_version": EVALUATOR_VERSION,
"portfolio_id": portfolio["portfolio_id"],
"portfolio_kind": portfolio_kind,
"portfolio_file_sha256": file_hash,
"portfolio_canonical_json_sha256": portfolio_canonical_hash,
"claim_boundary": claim_boundary,
@@ -1978,5 +2464,11 @@ def evaluate_cases(path: Path, allowed_sample_ids: set[str]) -> dict[str, Any]:
"results": results,
"subgroups": subgroup_report(results),
"failures": failures,
"failure_taxonomy": {
"primary_codes": FAILURE_TAXONOMY,
"contexts": FAILURE_CONTEXTS,
"claim_boundary": "diagnostic classification; not a release decision",
},
"portfolio_metrics": portfolio_metrics,
"results_canonical_json_sha256": results_hash,
}