1983 lines
72 KiB
Python
1983 lines
72 KiB
Python
"""Fail-closed, task-aware metrics for the GeoIntel Phase 4 benchmark.
|
|
|
|
The bundled portfolio is a synthetic contract fixture. Nothing emitted by
|
|
this module is, by itself, evidence of production-model accuracy.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import json
|
|
import math
|
|
from collections import defaultdict
|
|
from pathlib import Path
|
|
from statistics import mean, median
|
|
from typing import Any, Callable, Iterable
|
|
|
|
EVALUATOR_VERSION = "2.0.0"
|
|
REPORT_SCHEMA_VERSION = 2
|
|
SUBGROUP_MIN_CASE_SUPPORT = 5
|
|
CANONICAL_JSON_SPEC = (
|
|
"UTF-8 JSON produced with sort_keys=true, separators=(',', ':'), "
|
|
"ensure_ascii=false and allow_nan=false"
|
|
)
|
|
|
|
TASKS = {
|
|
"object_detection",
|
|
"footprint_segmentation",
|
|
"raster_classification",
|
|
"vector_comparison",
|
|
"change_detection",
|
|
"terrain_interpretation",
|
|
"geospatial_data_validation",
|
|
}
|
|
VALID_ANOMALY_SEVERITIES = {
|
|
"blocker",
|
|
"critical",
|
|
"major",
|
|
"minor",
|
|
"informational",
|
|
}
|
|
ERROR_CODES = {
|
|
"false_positive": "M-FP-CONFUSER",
|
|
"false_negative": "M-FN-MISSED",
|
|
"event_false_positive": "M-FP-CONFUSER",
|
|
"event_false_negative": "M-FN-MISSED",
|
|
"validation_false_positive": "D-VALIDATION-FP",
|
|
"validation_false_negative": "D-VALIDATION-FN",
|
|
"terrain_missing": "P-PARTIAL",
|
|
}
|
|
|
|
|
|
def canonical_json_bytes(value: Any) -> bytes:
|
|
"""Return the exact canonical byte representation used by all hashes."""
|
|
|
|
return json.dumps(
|
|
value,
|
|
ensure_ascii=False,
|
|
sort_keys=True,
|
|
separators=(",", ":"),
|
|
allow_nan=False,
|
|
).encode("utf-8")
|
|
|
|
|
|
def canonical_hash(value: Any) -> str:
|
|
return hashlib.sha256(canonical_json_bytes(value)).hexdigest()
|
|
|
|
|
|
def file_sha256(path: Path) -> str:
|
|
digest = hashlib.sha256()
|
|
with path.open("rb") as stream:
|
|
for chunk in iter(lambda: stream.read(1024 * 1024), b""):
|
|
digest.update(chunk)
|
|
return digest.hexdigest()
|
|
|
|
|
|
def safe_rate(numerator: int | float, denominator: int | float) -> float | None:
|
|
return float(numerator) / float(denominator) if denominator else None
|
|
|
|
|
|
def f1_score(precision: float | None, recall: float | None) -> float | None:
|
|
if precision is None or recall is None or precision + recall == 0:
|
|
return None
|
|
return 2 * precision * recall / (precision + recall)
|
|
|
|
|
|
def count_metrics(tp: int, fp: int, fn: int) -> dict[str, Any]:
|
|
precision = safe_rate(tp, tp + fp)
|
|
recall = safe_rate(tp, tp + fn)
|
|
return {
|
|
"true_positive": tp,
|
|
"false_positive": fp,
|
|
"false_negative": fn,
|
|
"prediction_count": tp + fp,
|
|
"reference_count": tp + fn,
|
|
"precision": precision,
|
|
"recall": recall,
|
|
"f1": f1_score(precision, recall),
|
|
"false_discovery_rate": safe_rate(fp, tp + fp),
|
|
"miss_rate": safe_rate(fn, tp + fn),
|
|
"precision_ci95_wilson": wilson_interval(tp, tp + fp),
|
|
"recall_ci95_wilson": wilson_interval(tp, tp + fn),
|
|
}
|
|
|
|
|
|
def wilson_interval(
|
|
successes: int, total: int, z: float = 1.959963984540054
|
|
) -> dict[str, Any]:
|
|
if total <= 0:
|
|
return {"status": "undefined", "lower": None, "upper": None, "support": total}
|
|
proportion = successes / total
|
|
denominator = 1 + z * z / total
|
|
centre = (proportion + z * z / (2 * total)) / denominator
|
|
margin = (
|
|
z
|
|
* math.sqrt((proportion * (1 - proportion) + z * z / (4 * total)) / total)
|
|
/ denominator
|
|
)
|
|
return {
|
|
"status": "computed",
|
|
"lower": max(0.0, centre - margin),
|
|
"upper": min(1.0, centre + margin),
|
|
"support": total,
|
|
"method": "Wilson score interval",
|
|
}
|
|
|
|
|
|
def _finite_float(value: Any, label: str) -> float:
|
|
if isinstance(value, bool) or not isinstance(value, (int, float)):
|
|
raise ValueError(f"{label} must be a finite number")
|
|
converted = float(value)
|
|
if not math.isfinite(converted):
|
|
raise ValueError(f"{label} must be a finite number")
|
|
return converted
|
|
|
|
|
|
def _probability(value: Any, label: str) -> float:
|
|
converted = _finite_float(value, label)
|
|
if not 0.0 <= converted <= 1.0:
|
|
raise ValueError(f"{label} must be between 0 and 1")
|
|
return converted
|
|
|
|
|
|
def _nonempty_mapping(value: Any, label: str) -> dict[str, Any]:
|
|
if not isinstance(value, dict) or not value:
|
|
raise ValueError(f"{label} must be a non-empty object")
|
|
return value
|
|
|
|
|
|
def _validate_case_common(case: dict[str, Any]) -> None:
|
|
sample_id = case.get("sample_id")
|
|
if not isinstance(sample_id, str) or not sample_id.strip():
|
|
raise ValueError("Every case requires a non-empty sample_id")
|
|
if case.get("task") not in TASKS:
|
|
raise ValueError(f"{sample_id}: unsupported task {case.get('task')!r}")
|
|
if not isinstance(case.get("metadata"), dict):
|
|
raise ValueError(f"{sample_id}: metadata must be an object")
|
|
if not isinstance(case.get("config"), dict):
|
|
raise ValueError(f"{sample_id}: config must be an explicit object")
|
|
_nonempty_mapping(case.get("lineage"), f"{sample_id}: lineage")
|
|
|
|
|
|
def _stable_id(item: dict[str, Any], label: str) -> str:
|
|
identifier = item.get("id")
|
|
if not isinstance(identifier, str) or not identifier.strip():
|
|
raise ValueError(f"{label} requires a non-empty string id")
|
|
return identifier
|
|
|
|
|
|
def _validate_unique_ids(items: list[dict[str, Any]], label: str) -> None:
|
|
identifiers = [
|
|
_stable_id(item, f"{label}[{index}]") for index, item in enumerate(items)
|
|
]
|
|
duplicates = sorted(
|
|
{identifier for identifier in identifiers if identifiers.count(identifier) > 1}
|
|
)
|
|
if duplicates:
|
|
raise ValueError(f"{label} contains duplicate ids: {duplicates}")
|
|
|
|
|
|
def _validate_bbox(value: Any, label: str) -> list[float]:
|
|
if not isinstance(value, list) or len(value) != 4:
|
|
raise ValueError(f"{label} must be [min_x, min_y, max_x, max_y]")
|
|
bbox = [
|
|
_finite_float(coordinate, f"{label}[{index}]")
|
|
for index, coordinate in enumerate(value)
|
|
]
|
|
if bbox[2] <= bbox[0] or bbox[3] <= bbox[1]:
|
|
raise ValueError(f"{label} must have positive width and height")
|
|
return bbox
|
|
|
|
|
|
def _validate_classes(case: dict[str, Any]) -> list[str | int]:
|
|
sample_id = case["sample_id"]
|
|
classes = case.get("classes")
|
|
if not isinstance(classes, list) or not classes:
|
|
raise ValueError(f"{sample_id}: classes must be a non-empty list")
|
|
if any(
|
|
isinstance(value, bool) or not isinstance(value, (str, int))
|
|
for value in classes
|
|
):
|
|
raise ValueError(f"{sample_id}: classes may contain only strings or integers")
|
|
if len({canonical_hash(value) for value in classes}) != len(classes):
|
|
raise ValueError(f"{sample_id}: classes must be unique")
|
|
if len({type(value) for value in classes}) != 1:
|
|
raise ValueError(f"{sample_id}: class values must use one JSON scalar type")
|
|
return classes
|
|
|
|
|
|
def _validate_labeled_items(
|
|
case: dict[str, Any],
|
|
items: list[dict[str, Any]],
|
|
label: str,
|
|
) -> None:
|
|
classes = _validate_classes(case)
|
|
for index, item in enumerate(items):
|
|
if item.get("class") not in classes:
|
|
raise ValueError(
|
|
f"{case['sample_id']}: {label}[{index}].class is outside the "
|
|
"declared ontology"
|
|
)
|
|
|
|
|
|
def _class_compatible(left: dict[str, Any], right: dict[str, Any]) -> bool:
|
|
return left.get("class") == right.get("class")
|
|
|
|
|
|
def bbox_iou(left: list[float], right: list[float]) -> float:
|
|
x1, y1 = max(left[0], right[0]), max(left[1], right[1])
|
|
x2, y2 = min(left[2], right[2]), min(left[3], right[3])
|
|
intersection = max(0.0, x2 - x1) * max(0.0, y2 - y1)
|
|
left_area = max(0.0, left[2] - left[0]) * max(0.0, left[3] - left[1])
|
|
right_area = max(0.0, right[2] - right[0]) * max(0.0, right[3] - right[1])
|
|
union = left_area + right_area - intersection
|
|
return intersection / union if union > 0 else 0.0
|
|
|
|
|
|
def greedy_match(
|
|
predictions: list[dict[str, Any]],
|
|
references: list[dict[str, Any]],
|
|
overlap: Callable[[dict[str, Any], dict[str, Any]], float],
|
|
threshold: float,
|
|
*,
|
|
class_aware: bool = False,
|
|
) -> tuple[list[dict[str, Any]], list[dict[str, Any]], list[dict[str, Any]]]:
|
|
unmatched = set(range(len(references)))
|
|
matches: list[dict[str, Any]] = []
|
|
false_positives: list[dict[str, Any]] = []
|
|
ordered = sorted(
|
|
predictions,
|
|
key=lambda item: (
|
|
-float(item.get("confidence", 1.0)),
|
|
_stable_id(item, "prediction"),
|
|
),
|
|
)
|
|
for prediction in ordered:
|
|
candidates: list[tuple[float, str, int]] = []
|
|
for index in sorted(unmatched):
|
|
reference = references[index]
|
|
if class_aware and not _class_compatible(prediction, reference):
|
|
continue
|
|
candidates.append(
|
|
(
|
|
overlap(prediction, reference),
|
|
_stable_id(reference, "reference"),
|
|
index,
|
|
)
|
|
)
|
|
if candidates:
|
|
score, _reference_id, index = min(
|
|
candidates, key=lambda item: (-item[0], item[1])
|
|
)
|
|
else:
|
|
score, index = 0.0, -1
|
|
if index >= 0 and score >= threshold:
|
|
unmatched.remove(index)
|
|
match = {
|
|
"prediction_id": prediction["id"],
|
|
"reference_id": references[index]["id"],
|
|
"overlap": score,
|
|
"confidence": prediction.get("confidence"),
|
|
}
|
|
if class_aware:
|
|
match["class"] = prediction["class"]
|
|
matches.append(match)
|
|
else:
|
|
false_positives.append(prediction)
|
|
false_negatives = [references[index] for index in sorted(unmatched)]
|
|
return matches, false_positives, false_negatives
|
|
|
|
|
|
def interpolated_ap(
|
|
points: list[tuple[float, str, int]],
|
|
reference_count: int,
|
|
) -> float | None:
|
|
if reference_count <= 0:
|
|
return None
|
|
ordered = sorted(points, key=lambda item: (-item[0], item[1]))
|
|
true_positive = 0
|
|
false_positive = 0
|
|
curve: list[tuple[float, float]] = []
|
|
for _confidence, _stable_prediction_id, correct in ordered:
|
|
true_positive += correct
|
|
false_positive += 1 - correct
|
|
curve.append(
|
|
(
|
|
true_positive / reference_count,
|
|
true_positive / (true_positive + false_positive),
|
|
)
|
|
)
|
|
values = []
|
|
for step in range(101):
|
|
recall_level = step / 100
|
|
values.append(
|
|
max(
|
|
(precision for recall, precision in curve if recall >= recall_level),
|
|
default=0.0,
|
|
)
|
|
)
|
|
return sum(values) / len(values)
|
|
|
|
|
|
def detection_ap(
|
|
predictions: list[dict[str, Any]],
|
|
references: list[dict[str, Any]],
|
|
iou_threshold: float,
|
|
) -> float | None:
|
|
unmatched = set(range(len(references)))
|
|
points: list[tuple[float, str, int]] = []
|
|
ordered = sorted(
|
|
predictions,
|
|
key=lambda item: (
|
|
-float(item["confidence"]),
|
|
_stable_id(item, "prediction"),
|
|
),
|
|
)
|
|
for prediction in ordered:
|
|
candidates: list[tuple[float, str, int]] = []
|
|
for index in sorted(unmatched):
|
|
reference = references[index]
|
|
if not _class_compatible(prediction, reference):
|
|
continue
|
|
candidates.append(
|
|
(
|
|
bbox_iou(prediction["bbox"], reference["bbox"]),
|
|
_stable_id(reference, "reference"),
|
|
index,
|
|
)
|
|
)
|
|
if candidates:
|
|
score, _reference_id, index = min(
|
|
candidates, key=lambda item: (-item[0], item[1])
|
|
)
|
|
else:
|
|
score, index = 0.0, -1
|
|
correct = int(index >= 0 and score >= iou_threshold)
|
|
if correct:
|
|
unmatched.remove(index)
|
|
points.append((float(prediction["confidence"]), prediction["id"], correct))
|
|
return interpolated_ap(points, len(references))
|
|
|
|
|
|
def calibration_metrics(
|
|
predictions: list[dict[str, Any]], matches: list[dict[str, Any]], bins: int = 5
|
|
) -> dict[str, Any]:
|
|
matched_ids = {item["prediction_id"] for item in matches}
|
|
scored = [
|
|
(float(item["confidence"]), 1 if item.get("id") in matched_ids else 0)
|
|
for item in predictions
|
|
]
|
|
if not scored:
|
|
return {"ece": None, "brier": None, "bins": [], "status": "undefined"}
|
|
blocks = []
|
|
ece = 0.0
|
|
for index in range(bins):
|
|
lower = index / bins
|
|
upper = (index + 1) / bins
|
|
selected = [
|
|
(confidence, correct)
|
|
for confidence, correct in scored
|
|
if lower <= confidence <= upper
|
|
and (index == bins - 1 or confidence < upper)
|
|
]
|
|
if not selected:
|
|
blocks.append(
|
|
{
|
|
"lower": lower,
|
|
"upper": upper,
|
|
"count": 0,
|
|
"mean_confidence": None,
|
|
"accuracy": None,
|
|
}
|
|
)
|
|
continue
|
|
avg_confidence = mean(item[0] for item in selected)
|
|
accuracy = mean(item[1] for item in selected)
|
|
ece += len(selected) / len(scored) * abs(accuracy - avg_confidence)
|
|
blocks.append(
|
|
{
|
|
"lower": lower,
|
|
"upper": upper,
|
|
"count": len(selected),
|
|
"mean_confidence": avg_confidence,
|
|
"accuracy": accuracy,
|
|
}
|
|
)
|
|
return {
|
|
"status": "computed",
|
|
"ece": ece,
|
|
"brier": mean((confidence - correct) ** 2 for confidence, correct in scored),
|
|
"bins": blocks,
|
|
"binning": "five fixed equal-width bins",
|
|
}
|
|
|
|
|
|
def coverage_risk(
|
|
predictions: list[dict[str, Any]],
|
|
references: list[dict[str, Any]],
|
|
match_iou: float,
|
|
operating_threshold: float,
|
|
) -> list[dict[str, Any]]:
|
|
"""Report retention, reference coverage and risk including every FN."""
|
|
|
|
thresholds = sorted({0.0, 0.5, 0.7, 0.9, operating_threshold})
|
|
rows = []
|
|
for threshold in thresholds:
|
|
retained = [
|
|
item for item in predictions if float(item["confidence"]) >= threshold
|
|
]
|
|
matches, false_positives, false_negatives = greedy_match(
|
|
retained,
|
|
references,
|
|
lambda prediction, reference: bbox_iou(
|
|
prediction["bbox"], reference["bbox"]
|
|
),
|
|
match_iou,
|
|
class_aware=True,
|
|
)
|
|
tp = len(matches)
|
|
fp = len(false_positives)
|
|
fn = len(false_negatives)
|
|
rows.append(
|
|
{
|
|
"threshold": threshold,
|
|
"retained_prediction_count": len(retained),
|
|
"total_prediction_count": len(predictions),
|
|
"retained_prediction_coverage": safe_rate(
|
|
len(retained), len(predictions)
|
|
),
|
|
"matched_reference_count": tp,
|
|
"reference_count": len(references),
|
|
"reference_coverage": safe_rate(tp, len(references)),
|
|
"false_positive_count": fp,
|
|
"false_negative_count": fn,
|
|
"risk": safe_rate(fp + fn, tp + fp + fn),
|
|
"risk_definition": (
|
|
"(false_positive + false_negative) / "
|
|
"(true_positive + false_positive + false_negative)"
|
|
),
|
|
}
|
|
)
|
|
return rows
|
|
|
|
|
|
def _mean_or_none(values: Iterable[float | None]) -> float | None:
|
|
retained = [value for value in values if value is not None]
|
|
return mean(retained) if retained else None
|
|
|
|
|
|
def _validate_detection_case(
|
|
case: dict[str, Any],
|
|
) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
|
|
references = case.get("references")
|
|
predictions = case.get("predictions")
|
|
if not isinstance(references, list) or not all(
|
|
isinstance(item, dict) for item in references
|
|
):
|
|
raise ValueError(f"{case['sample_id']}: references must be an object list")
|
|
if not isinstance(predictions, list) or not all(
|
|
isinstance(item, dict) for item in predictions
|
|
):
|
|
raise ValueError(f"{case['sample_id']}: predictions must be an object list")
|
|
_validate_unique_ids(references, f"{case['sample_id']}.references")
|
|
_validate_unique_ids(predictions, f"{case['sample_id']}.predictions")
|
|
_validate_labeled_items(case, references, "references")
|
|
_validate_labeled_items(case, predictions, "predictions")
|
|
for index, item in enumerate(references):
|
|
_validate_bbox(
|
|
item.get("bbox"),
|
|
f"{case['sample_id']}.references[{index}].bbox",
|
|
)
|
|
for index, item in enumerate(predictions):
|
|
_validate_bbox(
|
|
item.get("bbox"),
|
|
f"{case['sample_id']}.predictions[{index}].bbox",
|
|
)
|
|
_probability(
|
|
item.get("confidence"),
|
|
f"{case['sample_id']}.predictions[{index}].confidence",
|
|
)
|
|
return references, predictions
|
|
|
|
|
|
def evaluate_object_detection(case: dict[str, Any]) -> dict[str, Any]:
|
|
references, predictions = _validate_detection_case(case)
|
|
threshold = _probability(
|
|
case["config"].get("confidence_threshold"),
|
|
f"{case['sample_id']}.config.confidence_threshold",
|
|
)
|
|
match_iou = _probability(
|
|
case["config"].get("match_iou"),
|
|
f"{case['sample_id']}.config.match_iou",
|
|
)
|
|
retained = [item for item in predictions if float(item["confidence"]) >= threshold]
|
|
matches, false_positives, false_negatives = greedy_match(
|
|
retained,
|
|
references,
|
|
lambda prediction, reference: bbox_iou(prediction["bbox"], reference["bbox"]),
|
|
match_iou,
|
|
class_aware=True,
|
|
)
|
|
per_class = {}
|
|
for class_value in case["classes"]:
|
|
class_predictions = [item for item in retained if item["class"] == class_value]
|
|
all_class_predictions = [
|
|
item for item in predictions if item["class"] == class_value
|
|
]
|
|
class_references = [item for item in references if item["class"] == class_value]
|
|
class_matches, class_fp, class_fn = greedy_match(
|
|
class_predictions,
|
|
class_references,
|
|
lambda prediction, reference: bbox_iou(
|
|
prediction["bbox"], reference["bbox"]
|
|
),
|
|
match_iou,
|
|
class_aware=True,
|
|
)
|
|
class_metrics = count_metrics(len(class_matches), len(class_fp), len(class_fn))
|
|
class_metrics["ap50"] = detection_ap(
|
|
all_class_predictions, class_references, 0.5
|
|
)
|
|
per_class[str(class_value)] = class_metrics
|
|
metrics = count_metrics(len(matches), len(false_positives), len(false_negatives))
|
|
metrics.update(
|
|
{
|
|
"operating_confidence": threshold,
|
|
"match_iou": match_iou,
|
|
"matched_iou": distribution([item["overlap"] for item in matches]),
|
|
"ap50": detection_ap(predictions, references, 0.5),
|
|
"ap50_95": _mean_or_none(
|
|
detection_ap(predictions, references, 0.5 + step * 0.05)
|
|
for step in range(10)
|
|
),
|
|
"per_class": per_class,
|
|
"calibration": calibration_metrics(retained, matches),
|
|
"coverage_risk": coverage_risk(
|
|
predictions, references, match_iou, threshold
|
|
),
|
|
}
|
|
)
|
|
return result(
|
|
case,
|
|
metrics,
|
|
matches,
|
|
false_positives,
|
|
false_negatives,
|
|
post_filter_predictions=retained,
|
|
filter_description={
|
|
"applied": True,
|
|
"field": "confidence",
|
|
"operator": ">=",
|
|
"threshold": threshold,
|
|
},
|
|
)
|
|
|
|
|
|
def polygon(item: dict[str, Any]):
|
|
from shapely.geometry import Polygon
|
|
|
|
return Polygon(item["polygon"])
|
|
|
|
|
|
def distribution(values: list[float]) -> dict[str, Any]:
|
|
if not values:
|
|
return {"count": 0, "mean": None, "median": None, "min": None, "max": None}
|
|
if any(not math.isfinite(value) for value in values):
|
|
raise ValueError("Distribution values must be finite")
|
|
ordered = sorted(values)
|
|
return {
|
|
"count": len(values),
|
|
"mean": mean(values),
|
|
"median": median(values),
|
|
"min": ordered[0],
|
|
"max": ordered[-1],
|
|
}
|
|
|
|
|
|
def _validate_metric_spatial_context(case: dict[str, Any]) -> dict[str, Any]:
|
|
context = _nonempty_mapping(
|
|
case.get("spatial_context"),
|
|
f"{case['sample_id']}: spatial_context",
|
|
)
|
|
if context.get("metric") is not True:
|
|
raise ValueError(f"{case['sample_id']}: spatial_context.metric must be true")
|
|
if context.get("coordinate_units") != "m":
|
|
raise ValueError(
|
|
f"{case['sample_id']}: spatial_context.coordinate_units must be 'm'"
|
|
)
|
|
crs_value = context.get("crs")
|
|
if not isinstance(crs_value, str) or not crs_value.strip():
|
|
raise ValueError(f"{case['sample_id']}: a non-empty CRS is required")
|
|
try:
|
|
from pyproj import CRS
|
|
|
|
crs = CRS.from_user_input(crs_value)
|
|
except Exception as exc: # noqa: BLE001 - invalid CRS must fail closed
|
|
raise ValueError(f"{case['sample_id']}: invalid CRS {crs_value!r}") from exc
|
|
if not crs.is_projected:
|
|
raise ValueError(
|
|
f"{case['sample_id']}: metric geometry requires a projected CRS"
|
|
)
|
|
if not crs.axis_info or any(
|
|
axis.unit_conversion_factor is None
|
|
or not math.isclose(
|
|
axis.unit_conversion_factor,
|
|
1.0,
|
|
rel_tol=0.0,
|
|
abs_tol=1e-12,
|
|
)
|
|
for axis in crs.axis_info[:2]
|
|
):
|
|
raise ValueError(f"{case['sample_id']}: CRS axes must use metres")
|
|
return context
|
|
|
|
|
|
def _validated_polygon(item: dict[str, Any], label: str):
|
|
coordinates = item.get("polygon")
|
|
if not isinstance(coordinates, list) or len(coordinates) < 4:
|
|
raise ValueError(
|
|
f"{label}.polygon must contain a closed ring with at least four points"
|
|
)
|
|
normalized: list[tuple[float, float]] = []
|
|
for index, point in enumerate(coordinates):
|
|
if not isinstance(point, list) or len(point) != 2:
|
|
raise ValueError(f"{label}.polygon[{index}] must be an [x, y] pair")
|
|
normalized.append(
|
|
(
|
|
_finite_float(point[0], f"{label}.polygon[{index}][0]"),
|
|
_finite_float(point[1], f"{label}.polygon[{index}][1]"),
|
|
)
|
|
)
|
|
if normalized[0] != normalized[-1]:
|
|
raise ValueError(f"{label}.polygon ring must be explicitly closed")
|
|
from shapely.geometry import Polygon
|
|
|
|
geometry = Polygon(normalized)
|
|
if geometry.is_empty or geometry.area <= 0 or not geometry.is_valid:
|
|
raise ValueError(f"{label}.polygon must be non-empty, positive-area and valid")
|
|
return geometry
|
|
|
|
|
|
def _validated_polygon_items(
|
|
case: dict[str, Any],
|
|
) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
|
|
_validate_metric_spatial_context(case)
|
|
references = case.get("references")
|
|
predictions = case.get("predictions")
|
|
if not isinstance(references, list) or not all(
|
|
isinstance(item, dict) for item in references
|
|
):
|
|
raise ValueError(f"{case['sample_id']}: references must be an object list")
|
|
if not isinstance(predictions, list) or not all(
|
|
isinstance(item, dict) for item in predictions
|
|
):
|
|
raise ValueError(f"{case['sample_id']}: predictions must be an object list")
|
|
_validate_unique_ids(references, f"{case['sample_id']}.references")
|
|
_validate_unique_ids(predictions, f"{case['sample_id']}.predictions")
|
|
_validate_labeled_items(case, references, "references")
|
|
_validate_labeled_items(case, predictions, "predictions")
|
|
validated_references = [
|
|
dict(
|
|
item,
|
|
geometry=_validated_polygon(
|
|
item, f"{case['sample_id']}.references[{index}]"
|
|
),
|
|
)
|
|
for index, item in enumerate(references)
|
|
]
|
|
validated_predictions = [
|
|
dict(
|
|
item,
|
|
geometry=_validated_polygon(
|
|
item, f"{case['sample_id']}.predictions[{index}]"
|
|
),
|
|
)
|
|
for index, item in enumerate(predictions)
|
|
]
|
|
return validated_references, validated_predictions
|
|
|
|
|
|
def _polygon_overlap(prediction: dict[str, Any], reference: dict[str, Any]) -> float:
|
|
intersection = prediction["geometry"].intersection(reference["geometry"]).area
|
|
union = prediction["geometry"].union(reference["geometry"]).area
|
|
return intersection / union if union > 0 else 0.0
|
|
|
|
|
|
def boundary_f1(prediction, reference, tolerance: float) -> float | None:
|
|
predicted_boundary = prediction.boundary
|
|
reference_boundary = reference.boundary
|
|
predicted_length = predicted_boundary.length
|
|
reference_length = reference_boundary.length
|
|
if predicted_length <= 0 or reference_length <= 0:
|
|
return None
|
|
precision = (
|
|
predicted_boundary.intersection(reference_boundary.buffer(tolerance)).length
|
|
/ predicted_length
|
|
)
|
|
recall = (
|
|
reference_boundary.intersection(predicted_boundary.buffer(tolerance)).length
|
|
/ reference_length
|
|
)
|
|
return f1_score(precision, recall)
|
|
|
|
|
|
def evaluate_footprint_segmentation(case: dict[str, Any]) -> dict[str, Any]:
|
|
references, predictions = _validated_polygon_items(case)
|
|
match_iou = _probability(
|
|
case["config"].get("match_iou"),
|
|
f"{case['sample_id']}.config.match_iou",
|
|
)
|
|
tolerance = _finite_float(
|
|
case["config"].get("boundary_tolerance_m"),
|
|
f"{case['sample_id']}.config.boundary_tolerance_m",
|
|
)
|
|
if tolerance <= 0:
|
|
raise ValueError(f"{case['sample_id']}: boundary tolerance must be positive")
|
|
for index, item in enumerate(predictions):
|
|
if "confidence" in item:
|
|
_probability(
|
|
item["confidence"],
|
|
f"{case['sample_id']}.predictions[{index}].confidence",
|
|
)
|
|
matches, false_positives, false_negatives = greedy_match(
|
|
predictions,
|
|
references,
|
|
_polygon_overlap,
|
|
match_iou,
|
|
class_aware=True,
|
|
)
|
|
by_prediction = {item["id"]: item for item in predictions}
|
|
by_reference = {item["id"]: item for item in references}
|
|
dice = []
|
|
boundary = []
|
|
centroids = []
|
|
area_errors = []
|
|
for match in matches:
|
|
pred = by_prediction[match["prediction_id"]]["geometry"]
|
|
ref = by_reference[match["reference_id"]]["geometry"]
|
|
intersection = pred.intersection(ref).area
|
|
dice.append(2 * intersection / (pred.area + ref.area))
|
|
boundary.append(boundary_f1(pred, ref, tolerance))
|
|
centroids.append(pred.centroid.distance(ref.centroid))
|
|
area_errors.append((pred.area - ref.area) / ref.area)
|
|
metrics = count_metrics(len(matches), len(false_positives), len(false_negatives))
|
|
metrics.update(
|
|
{
|
|
"mean_iou": _mean_or_none([item["overlap"] for item in matches]),
|
|
"mean_dice": _mean_or_none(dice),
|
|
"mean_boundary_f1": _mean_or_none(boundary),
|
|
"centroid_distance_m": distribution(centroids),
|
|
"relative_area_error": distribution(area_errors),
|
|
"topologically_valid_predictions": len(predictions),
|
|
"topologically_invalid_predictions": 0,
|
|
"spatial_context_validated": True,
|
|
}
|
|
)
|
|
return result(
|
|
case,
|
|
metrics,
|
|
matches,
|
|
false_positives,
|
|
false_negatives,
|
|
)
|
|
|
|
|
|
def _validate_rectangular_grid(
|
|
value: Any,
|
|
label: str,
|
|
) -> tuple[list[list[Any]], tuple[int, int]]:
|
|
if (
|
|
not isinstance(value, list)
|
|
or not value
|
|
or not all(isinstance(row, list) for row in value)
|
|
):
|
|
raise ValueError(f"{label} must be a non-empty two-dimensional array")
|
|
width = len(value[0])
|
|
if width <= 0 or any(len(row) != width for row in value):
|
|
raise ValueError(f"{label} must be exactly rectangular")
|
|
return value, (len(value), width)
|
|
|
|
|
|
def _validate_raster_side(
|
|
case: dict[str, Any],
|
|
side_name: str,
|
|
expected_shape: tuple[int, int],
|
|
) -> dict[str, Any]:
|
|
context = _nonempty_mapping(
|
|
case["raster_context"].get(side_name),
|
|
f"{case['sample_id']}.raster_context.{side_name}",
|
|
)
|
|
required = {"crs", "transform", "shape", "nodata", "mask"}
|
|
missing = sorted(required - context.keys())
|
|
if missing:
|
|
raise ValueError(
|
|
f"{case['sample_id']}.raster_context.{side_name} missing {missing}"
|
|
)
|
|
crs_value = context["crs"]
|
|
if not isinstance(crs_value, str) or not crs_value.strip():
|
|
raise ValueError(f"{case['sample_id']}: raster CRS must be non-empty")
|
|
try:
|
|
from pyproj import CRS
|
|
|
|
normalized_crs = CRS.from_user_input(crs_value)
|
|
except Exception as exc: # noqa: BLE001 - invalid CRS must fail closed
|
|
raise ValueError(
|
|
f"{case['sample_id']}: invalid raster CRS {crs_value!r}"
|
|
) from exc
|
|
transform = context["transform"]
|
|
if not isinstance(transform, list) or len(transform) != 6:
|
|
raise ValueError(
|
|
f"{case['sample_id']}: raster transform must contain six numbers"
|
|
)
|
|
normalized_transform = tuple(
|
|
_finite_float(
|
|
value,
|
|
f"{case['sample_id']}.{side_name}.transform[{index}]",
|
|
)
|
|
for index, value in enumerate(transform)
|
|
)
|
|
shape = context["shape"]
|
|
if (
|
|
not isinstance(shape, list)
|
|
or len(shape) != 2
|
|
or any(
|
|
isinstance(value, bool) or not isinstance(value, int) or value <= 0
|
|
for value in shape
|
|
)
|
|
):
|
|
raise ValueError(
|
|
f"{case['sample_id']}: raster shape must be [positive rows, "
|
|
"positive columns]"
|
|
)
|
|
if tuple(shape) != expected_shape:
|
|
raise ValueError(
|
|
f"{case['sample_id']}: declared {side_name} shape differs from the grid"
|
|
)
|
|
mask, mask_shape = _validate_rectangular_grid(
|
|
context["mask"],
|
|
f"{case['sample_id']}.{side_name}.mask",
|
|
)
|
|
if mask_shape != expected_shape or any(
|
|
not isinstance(value, bool) for row in mask for value in row
|
|
):
|
|
raise ValueError(
|
|
f"{case['sample_id']}: {side_name} mask must be a boolean grid "
|
|
"with the exact raster shape"
|
|
)
|
|
nodata = context["nodata"]
|
|
if nodata is not None:
|
|
_finite_float(nodata, f"{case['sample_id']}.{side_name}.nodata")
|
|
return {
|
|
"crs": normalized_crs,
|
|
"transform": normalized_transform,
|
|
"shape": expected_shape,
|
|
"nodata": nodata,
|
|
"mask": mask,
|
|
}
|
|
|
|
|
|
def _is_nodata(value: Any, nodata: Any) -> bool:
|
|
return nodata is not None and value == nodata
|
|
|
|
|
|
def _validate_raster_context(
|
|
case: dict[str, Any],
|
|
reference_shape: tuple[int, int],
|
|
prediction_shape: tuple[int, int],
|
|
) -> tuple[dict[str, Any], dict[str, Any]]:
|
|
if reference_shape != prediction_shape:
|
|
raise ValueError(f"{case['sample_id']}: raster shapes differ")
|
|
_nonempty_mapping(
|
|
case.get("raster_context"),
|
|
f"{case['sample_id']}: raster_context",
|
|
)
|
|
reference = _validate_raster_side(case, "reference", reference_shape)
|
|
prediction = _validate_raster_side(case, "prediction", prediction_shape)
|
|
if reference["crs"] != prediction["crs"]:
|
|
raise ValueError(f"{case['sample_id']}: raster CRS alignment differs")
|
|
if reference["transform"] != prediction["transform"]:
|
|
raise ValueError(f"{case['sample_id']}: raster affine alignment differs")
|
|
if reference["shape"] != prediction["shape"]:
|
|
raise ValueError(f"{case['sample_id']}: raster shape alignment differs")
|
|
return reference, prediction
|
|
|
|
|
|
def evaluate_raster_classification(case: dict[str, Any]) -> dict[str, Any]:
|
|
classes = _validate_classes(case)
|
|
reference_grid, reference_shape = _validate_rectangular_grid(
|
|
case.get("references"),
|
|
f"{case['sample_id']}.references",
|
|
)
|
|
prediction_grid, prediction_shape = _validate_rectangular_grid(
|
|
case.get("predictions"),
|
|
f"{case['sample_id']}.predictions",
|
|
)
|
|
reference_context, prediction_context = _validate_raster_context(
|
|
case,
|
|
reference_shape,
|
|
prediction_shape,
|
|
)
|
|
labels = [str(value) for value in classes]
|
|
missing_label = "__prediction_nodata__"
|
|
confusion = {
|
|
reference: {prediction: 0 for prediction in [*labels, missing_label]}
|
|
for reference in labels
|
|
}
|
|
failures = []
|
|
evaluated_reference_count = 0
|
|
prediction_present_count = 0
|
|
ignored_reference_count = 0
|
|
prediction_outside_reference_count = 0
|
|
correct_count = 0
|
|
for row_index in range(reference_shape[0]):
|
|
for column_index in range(reference_shape[1]):
|
|
reference_value = reference_grid[row_index][column_index]
|
|
prediction_value = prediction_grid[row_index][column_index]
|
|
reference_valid = reference_context["mask"][row_index][column_index]
|
|
prediction_valid = prediction_context["mask"][row_index][column_index]
|
|
if reference_valid and _is_nodata(
|
|
reference_value, reference_context["nodata"]
|
|
):
|
|
raise ValueError(
|
|
f"{case['sample_id']}: reference mask marks nodata as "
|
|
f"valid at [{row_index}, {column_index}]"
|
|
)
|
|
if prediction_valid and _is_nodata(
|
|
prediction_value, prediction_context["nodata"]
|
|
):
|
|
raise ValueError(
|
|
f"{case['sample_id']}: prediction mask marks nodata as "
|
|
f"valid at [{row_index}, {column_index}]"
|
|
)
|
|
if not reference_valid:
|
|
ignored_reference_count += 1
|
|
if prediction_valid:
|
|
prediction_outside_reference_count += 1
|
|
continue
|
|
if reference_value not in classes:
|
|
raise ValueError(
|
|
f"{case['sample_id']}: reference class outside ontology "
|
|
f"at [{row_index}, {column_index}]"
|
|
)
|
|
evaluated_reference_count += 1
|
|
reference_key = str(reference_value)
|
|
if not prediction_valid:
|
|
confusion[reference_key][missing_label] += 1
|
|
failures.append(
|
|
{
|
|
"pixel_index": row_index * reference_shape[1] + column_index,
|
|
"row": row_index,
|
|
"column": column_index,
|
|
"reference": reference_value,
|
|
"prediction": None,
|
|
"reason": "prediction_masked_or_nodata",
|
|
}
|
|
)
|
|
continue
|
|
if prediction_value not in classes:
|
|
raise ValueError(
|
|
f"{case['sample_id']}: prediction class outside ontology "
|
|
f"at [{row_index}, {column_index}]"
|
|
)
|
|
prediction_present_count += 1
|
|
prediction_key = str(prediction_value)
|
|
confusion[reference_key][prediction_key] += 1
|
|
if reference_value == prediction_value:
|
|
correct_count += 1
|
|
else:
|
|
failures.append(
|
|
{
|
|
"pixel_index": row_index * reference_shape[1] + column_index,
|
|
"row": row_index,
|
|
"column": column_index,
|
|
"reference": reference_value,
|
|
"prediction": prediction_value,
|
|
"reason": "class_mismatch",
|
|
}
|
|
)
|
|
per_class = {}
|
|
for class_value in classes:
|
|
class_key = str(class_value)
|
|
tp = confusion[class_key][class_key]
|
|
fp = sum(
|
|
confusion[str(other)][class_key]
|
|
for other in classes
|
|
if other != class_value
|
|
)
|
|
fn = sum(
|
|
confusion[class_key][prediction]
|
|
for prediction in [*labels, missing_label]
|
|
if prediction != class_key
|
|
)
|
|
values = count_metrics(tp, fp, fn)
|
|
values["iou"] = safe_rate(tp, tp + fp + fn)
|
|
per_class[class_key] = values
|
|
metrics = {
|
|
"pixel_count": reference_shape[0] * reference_shape[1],
|
|
"evaluated_reference_pixel_count": evaluated_reference_count,
|
|
"prediction_present_pixel_count": prediction_present_count,
|
|
"ignored_reference_pixel_count": ignored_reference_count,
|
|
"prediction_outside_reference_count": (prediction_outside_reference_count),
|
|
"correct_pixel_count": correct_count,
|
|
"prediction_coverage": safe_rate(
|
|
prediction_present_count,
|
|
evaluated_reference_count,
|
|
),
|
|
"prediction_coverage_ci95_wilson": wilson_interval(
|
|
prediction_present_count,
|
|
evaluated_reference_count,
|
|
),
|
|
"accuracy": safe_rate(correct_count, evaluated_reference_count),
|
|
"accuracy_ci95_wilson": wilson_interval(
|
|
correct_count,
|
|
evaluated_reference_count,
|
|
),
|
|
"mean_iou": _mean_or_none([value["iou"] for value in per_class.values()]),
|
|
"macro_f1": _mean_or_none([value["f1"] for value in per_class.values()]),
|
|
"per_class": per_class,
|
|
"confusion_matrix": confusion,
|
|
"alignment_validated": True,
|
|
}
|
|
return result(case, metrics, [], failures, [])
|
|
|
|
|
|
def evaluate_vector_comparison(case: dict[str, Any]) -> dict[str, Any]:
|
|
references, predictions = _validated_polygon_items(case)
|
|
match_iou = _probability(
|
|
case["config"].get("match_iou"),
|
|
f"{case['sample_id']}.config.match_iou",
|
|
)
|
|
matches, false_positives, false_negatives = greedy_match(
|
|
predictions,
|
|
references,
|
|
_polygon_overlap,
|
|
match_iou,
|
|
class_aware=True,
|
|
)
|
|
metrics = count_metrics(len(matches), len(false_positives), len(false_negatives))
|
|
metrics.update(
|
|
{
|
|
"mean_iou": _mean_or_none([item["overlap"] for item in matches]),
|
|
"topologically_valid": True,
|
|
"spatial_context_validated": True,
|
|
}
|
|
)
|
|
return result(case, metrics, matches, false_positives, false_negatives)
|
|
|
|
|
|
def evaluate_change_detection(case: dict[str, Any]) -> dict[str, Any]:
|
|
references, predictions = _validate_detection_case(case)
|
|
match_iou = _probability(
|
|
case["config"].get("match_iou"),
|
|
f"{case['sample_id']}.config.match_iou",
|
|
)
|
|
matches = []
|
|
false_positives = []
|
|
false_negatives = []
|
|
event_metrics = {}
|
|
for event in case["classes"]:
|
|
event_predictions = [item for item in predictions if item["class"] == event]
|
|
event_references = [item for item in references if item["class"] == event]
|
|
event_matches, event_fp, event_fn = greedy_match(
|
|
event_predictions,
|
|
event_references,
|
|
lambda prediction, reference: bbox_iou(
|
|
prediction["bbox"], reference["bbox"]
|
|
),
|
|
match_iou,
|
|
class_aware=True,
|
|
)
|
|
matches.extend(dict(item, event=event) for item in event_matches)
|
|
false_positives.extend(event_fp)
|
|
false_negatives.extend(event_fn)
|
|
event_metrics[str(event)] = count_metrics(
|
|
len(event_matches), len(event_fp), len(event_fn)
|
|
)
|
|
metrics = count_metrics(len(matches), len(false_positives), len(false_negatives))
|
|
metrics["event_metrics"] = event_metrics
|
|
return result(case, metrics, matches, false_positives, false_negatives)
|
|
|
|
|
|
def evaluate_terrain(case: dict[str, Any]) -> dict[str, Any]:
|
|
references = case.get("references")
|
|
predictions = case.get("predictions")
|
|
if not isinstance(references, list) or not isinstance(predictions, list):
|
|
raise ValueError(f"{case['sample_id']}: terrain inputs must be lists")
|
|
if len(references) != len(predictions):
|
|
raise ValueError(f"{case['sample_id']}: terrain vector lengths differ")
|
|
units = case.get("units")
|
|
if not isinstance(units, str) or not units.strip():
|
|
raise ValueError(f"{case['sample_id']}: terrain units must be explicit")
|
|
normalized_references = [
|
|
_finite_float(value, f"{case['sample_id']}.references[{index}]")
|
|
for index, value in enumerate(references)
|
|
]
|
|
normalized_predictions: list[float | None] = []
|
|
for index, value in enumerate(predictions):
|
|
normalized_predictions.append(
|
|
None
|
|
if value is None
|
|
else _finite_float(
|
|
value,
|
|
f"{case['sample_id']}.predictions[{index}]",
|
|
)
|
|
)
|
|
pairs = [
|
|
(reference, prediction)
|
|
for reference, prediction in zip(
|
|
normalized_references,
|
|
normalized_predictions,
|
|
strict=True,
|
|
)
|
|
if prediction is not None
|
|
]
|
|
errors = [prediction - reference for reference, prediction in pairs]
|
|
missing = [
|
|
index for index, value in enumerate(normalized_predictions) if value is None
|
|
]
|
|
metrics = {
|
|
"units": units,
|
|
"reference_count": len(normalized_references),
|
|
"evaluated_count": len(pairs),
|
|
"missing_count": len(missing),
|
|
"coverage": safe_rate(len(pairs), len(normalized_references)),
|
|
"coverage_ci95_wilson": wilson_interval(len(pairs), len(normalized_references)),
|
|
"mae": _mean_or_none([abs(value) for value in errors]),
|
|
"rmse": (
|
|
math.sqrt(mean(value * value for value in errors)) if errors else None
|
|
),
|
|
"bias": _mean_or_none(errors),
|
|
"error_sum": sum(errors),
|
|
"absolute_error_sum": sum(abs(value) for value in errors),
|
|
"squared_error_sum": sum(value * value for value in errors),
|
|
"error_distribution": distribution(errors),
|
|
}
|
|
return result(
|
|
case,
|
|
metrics,
|
|
[],
|
|
[],
|
|
[{"missing_index": index} for index in missing],
|
|
)
|
|
|
|
|
|
def _normalized_anomalies(
|
|
case: dict[str, Any],
|
|
field: str,
|
|
) -> list[dict[str, str]]:
|
|
values = case.get(field)
|
|
if not isinstance(values, list):
|
|
raise ValueError(f"{case['sample_id']}: {field} must be a list")
|
|
normalized = []
|
|
for index, value in enumerate(values):
|
|
if not isinstance(value, dict):
|
|
raise ValueError(
|
|
f"{case['sample_id']}: {field}[{index}] must include code and severity"
|
|
)
|
|
code = value.get("code")
|
|
severity = value.get("severity")
|
|
if not isinstance(code, str) or not code.strip():
|
|
raise ValueError(f"{case['sample_id']}: {field}[{index}].code is invalid")
|
|
if severity not in VALID_ANOMALY_SEVERITIES:
|
|
raise ValueError(
|
|
f"{case['sample_id']}: {field}[{index}].severity is invalid"
|
|
)
|
|
normalized.append({"code": code, "severity": severity})
|
|
codes = [item["code"] for item in normalized]
|
|
duplicates = sorted({code for code in codes if codes.count(code) > 1})
|
|
if duplicates:
|
|
raise ValueError(f"{case['sample_id']}: duplicate {field} codes {duplicates}")
|
|
return normalized
|
|
|
|
|
|
def evaluate_validation(case: dict[str, Any]) -> dict[str, Any]:
|
|
expected_items = _normalized_anomalies(case, "expected_anomalies")
|
|
observed_items = _normalized_anomalies(case, "observed_anomalies")
|
|
expected = {item["code"]: item for item in expected_items}
|
|
observed = {item["code"]: item for item in observed_items}
|
|
matched_codes = sorted(expected.keys() & observed.keys())
|
|
false_positive_codes = sorted(observed.keys() - expected.keys())
|
|
false_negative_codes = sorted(expected.keys() - observed.keys())
|
|
matched = [
|
|
{
|
|
"anomaly": code,
|
|
"expected_severity": expected[code]["severity"],
|
|
"observed_severity": observed[code]["severity"],
|
|
}
|
|
for code in matched_codes
|
|
]
|
|
false_positives = [
|
|
{"anomaly": code, "severity": observed[code]["severity"]}
|
|
for code in false_positive_codes
|
|
]
|
|
false_negatives = [
|
|
{"anomaly": code, "severity": expected[code]["severity"]}
|
|
for code in false_negative_codes
|
|
]
|
|
metrics = count_metrics(len(matched), len(false_positives), len(false_negatives))
|
|
metrics["blocker_or_critical_miss_count"] = sum(
|
|
item["severity"] in {"blocker", "critical"} for item in false_negatives
|
|
)
|
|
metrics["misses_by_severity"] = {
|
|
severity: sum(item["severity"] == severity for item in false_negatives)
|
|
for severity in sorted(VALID_ANOMALY_SEVERITIES)
|
|
}
|
|
return result(case, metrics, matched, false_positives, false_negatives)
|
|
|
|
|
|
def _case_payload_fields(case: dict[str, Any]) -> tuple[str, str]:
|
|
if case["task"] == "geospatial_data_validation":
|
|
return "expected_anomalies", "observed_anomalies"
|
|
return "references", "predictions"
|
|
|
|
|
|
def result(
|
|
case: dict[str, Any],
|
|
metrics: dict[str, Any],
|
|
matches: list[dict[str, Any]],
|
|
false_positives: list[dict[str, Any]],
|
|
false_negatives: list[dict[str, Any]],
|
|
*,
|
|
post_filter_predictions: Any | None = None,
|
|
filter_description: dict[str, Any] | None = None,
|
|
) -> dict[str, Any]:
|
|
reference_field, prediction_field = _case_payload_fields(case)
|
|
exact_references = serializable(case[reference_field])
|
|
exact_predictions = serializable(case[prediction_field])
|
|
post_filter = (
|
|
exact_predictions
|
|
if post_filter_predictions is None
|
|
else serializable(post_filter_predictions)
|
|
)
|
|
case_hash = canonical_hash(case)
|
|
raw = {
|
|
"sample_id": case["sample_id"],
|
|
"task": case["task"],
|
|
"metadata": serializable(case["metadata"]),
|
|
"split": case["split"],
|
|
"config": serializable(case["config"]),
|
|
"input_lineage": serializable(case["lineage"]),
|
|
"reference_input_field": reference_field,
|
|
"prediction_input_field": prediction_field,
|
|
"references": exact_references,
|
|
"predictions_pre_filter": exact_predictions,
|
|
"predictions_post_filter": post_filter,
|
|
"filter": filter_description or {"applied": False},
|
|
"matches": serializable(matches),
|
|
"false_positives": serializable(false_positives),
|
|
"false_negatives": serializable(false_negatives),
|
|
"input_sha256": case_hash,
|
|
"hashes": {
|
|
"case_input_canonical_json_sha256": case_hash,
|
|
"references_canonical_json_sha256": canonical_hash(exact_references),
|
|
"predictions_pre_filter_canonical_json_sha256": canonical_hash(
|
|
exact_predictions
|
|
),
|
|
"predictions_post_filter_canonical_json_sha256": canonical_hash(
|
|
post_filter
|
|
),
|
|
"canonicalization": CANONICAL_JSON_SPEC,
|
|
},
|
|
}
|
|
failures = []
|
|
for item in raw["false_positives"]:
|
|
failures.append(failure_entry(case, "false_positive", item))
|
|
for item in raw["false_negatives"]:
|
|
failures.append(failure_entry(case, "false_negative", item))
|
|
return {
|
|
"sample_id": case["sample_id"],
|
|
"task": case["task"],
|
|
"metadata": serializable(case["metadata"]),
|
|
"metrics": metrics,
|
|
"raw": raw,
|
|
"failures": failures,
|
|
}
|
|
|
|
|
|
def serializable(value: Any) -> Any:
|
|
if isinstance(value, dict):
|
|
return {
|
|
key: serializable(item) for key, item in value.items() if key != "geometry"
|
|
}
|
|
if isinstance(value, list):
|
|
return [serializable(item) for item in value]
|
|
if isinstance(value, tuple):
|
|
return [serializable(item) for item in value]
|
|
return value
|
|
|
|
|
|
def failure_entry(
|
|
case: dict[str, Any], kind: str, evidence: dict[str, Any]
|
|
) -> dict[str, Any]:
|
|
if case["task"] == "change_detection":
|
|
kind = (
|
|
"event_false_positive"
|
|
if kind == "false_positive"
|
|
else "event_false_negative"
|
|
)
|
|
elif case["task"] == "geospatial_data_validation":
|
|
kind = (
|
|
"validation_false_positive"
|
|
if kind == "false_positive"
|
|
else "validation_false_negative"
|
|
)
|
|
elif case["task"] == "terrain_interpretation":
|
|
kind = "terrain_missing"
|
|
return {
|
|
"failure_id": canonical_hash(
|
|
{
|
|
"sample_id": case["sample_id"],
|
|
"kind": kind,
|
|
"evidence": serializable(evidence),
|
|
}
|
|
)[:20],
|
|
"sample_id": case["sample_id"],
|
|
"task": case["task"],
|
|
"error_code": ERROR_CODES[kind],
|
|
"kind": kind,
|
|
"metadata": case["metadata"],
|
|
"evidence": serializable(evidence),
|
|
}
|
|
|
|
|
|
EVALUATORS: dict[str, Callable[[dict[str, Any]], dict[str, Any]]] = {
|
|
"object_detection": evaluate_object_detection,
|
|
"footprint_segmentation": evaluate_footprint_segmentation,
|
|
"raster_classification": evaluate_raster_classification,
|
|
"vector_comparison": evaluate_vector_comparison,
|
|
"change_detection": evaluate_change_detection,
|
|
"terrain_interpretation": evaluate_terrain,
|
|
"geospatial_data_validation": evaluate_validation,
|
|
}
|
|
|
|
|
|
def _capability(
|
|
capability_id: str,
|
|
task: str,
|
|
implementation_paths: list[str],
|
|
implementation_kind: str,
|
|
evaluation_status: str,
|
|
suitable_metrics: list[str],
|
|
) -> dict[str, Any]:
|
|
return {
|
|
"capability_id": capability_id,
|
|
"task": task,
|
|
"implementation_paths": implementation_paths,
|
|
"implementation_kind": implementation_kind,
|
|
"evaluation_status": evaluation_status,
|
|
"suitable_metrics": suitable_metrics,
|
|
"claim_boundary": (
|
|
"A family mapping is not evidence that this capability has a "
|
|
"separate representative product benchmark."
|
|
),
|
|
}
|
|
|
|
|
|
def task_inventory() -> list[dict[str, Any]]:
|
|
"""Map concrete capabilities to evaluator families without overclaiming."""
|
|
|
|
synthetic = "synthetic_contract_case_only"
|
|
covered = "covered_by_family_not_separately_benchmarked"
|
|
separate = "not_separately_benchmarked"
|
|
return [
|
|
_capability(
|
|
"model_object_detection",
|
|
"object_detection",
|
|
["backend/app/services/detection_service.py"],
|
|
"model_inference_pipeline",
|
|
synthetic,
|
|
[
|
|
"precision",
|
|
"recall",
|
|
"F1",
|
|
"AP50",
|
|
"AP50-95",
|
|
"IoU",
|
|
"ECE",
|
|
"Brier",
|
|
"coverage-risk",
|
|
],
|
|
),
|
|
_capability(
|
|
"building_proposal_filtering",
|
|
"object_detection",
|
|
[
|
|
"scripts/train_building_proposal_classifier.py",
|
|
"scripts/evaluate_belgium_building_candidate.py",
|
|
],
|
|
"supporting_candidate_classifier",
|
|
separate,
|
|
["candidate precision", "candidate recall", "F1", "calibration"],
|
|
),
|
|
_capability(
|
|
"footprint_segmentation",
|
|
"footprint_segmentation",
|
|
[
|
|
"backend/app/services/segmentation_service.py",
|
|
"backend/app/services/segmentation_adapter.py",
|
|
],
|
|
"model_inference_pipeline",
|
|
synthetic,
|
|
[
|
|
"object precision",
|
|
"object recall",
|
|
"F1",
|
|
"IoU",
|
|
"Dice",
|
|
"boundary F1",
|
|
"centroid distance",
|
|
"area error",
|
|
"topology",
|
|
],
|
|
),
|
|
_capability(
|
|
"detection_and_vector_qa",
|
|
"vector_comparison",
|
|
[
|
|
"backend/app/services/qa_service.py",
|
|
"backend/app/services/detection_qa_service.py",
|
|
"backend/app/services/quality_check_service.py",
|
|
],
|
|
"deterministic_geospatial_comparison",
|
|
synthetic,
|
|
["precision", "recall", "F1", "IoU", "topology", "coverage"],
|
|
),
|
|
_capability(
|
|
"vector_clip_buffer_intersect",
|
|
"vector_comparison",
|
|
[
|
|
"backend/app/services/vector_operations_service.py",
|
|
"backend/app/services/vector_feature_service.py",
|
|
],
|
|
"deterministic_vector_processing",
|
|
covered,
|
|
[
|
|
"geometry validity",
|
|
"CRS correctness",
|
|
"area conservation",
|
|
"feature counts",
|
|
"topology",
|
|
],
|
|
),
|
|
_capability(
|
|
"temporal_vector_change",
|
|
"change_detection",
|
|
["backend/app/services/change_detection_service.py"],
|
|
"deterministic_change_detection",
|
|
synthetic,
|
|
["event precision", "event recall", "event F1", "IoU"],
|
|
),
|
|
_capability(
|
|
"thematic_raster_interpretation",
|
|
"raster_classification",
|
|
["backend/app/services/thematic_raster_analysis_service.py"],
|
|
"deterministic_source_interpretation",
|
|
"synthetic_metric_contract_only_no_generic_learned_classifier_claim",
|
|
["pixel accuracy", "per-class F1", "per-class IoU", "mean IoU"],
|
|
),
|
|
_capability(
|
|
"raster_clip_reproject_indices",
|
|
"raster_classification",
|
|
[
|
|
"backend/app/services/raster_service.py",
|
|
"backend/app/services/raster_operations_service.py",
|
|
],
|
|
"deterministic_raster_processing",
|
|
covered,
|
|
[
|
|
"CRS/transform preservation",
|
|
"pixel alignment",
|
|
"nodata",
|
|
"numeric tolerance",
|
|
],
|
|
),
|
|
_capability(
|
|
"raster_partition_mosaic",
|
|
"raster_classification",
|
|
["backend/app/services/raster_partition_analysis_service.py"],
|
|
"deterministic_raster_partitioning",
|
|
separate,
|
|
["seam equality", "coverage completeness", "resolution consistency"],
|
|
),
|
|
_capability(
|
|
"terrain_height_interpretation",
|
|
"terrain_interpretation",
|
|
[
|
|
"backend/app/services/terrain_analysis_service.py",
|
|
"backend/app/services/spw_terrain_service.py",
|
|
],
|
|
"deterministic_continuous_raster_analysis",
|
|
synthetic,
|
|
["MAE", "RMSE", "bias", "coverage", "unit integrity"],
|
|
),
|
|
_capability(
|
|
"flood_hazard_interpretation",
|
|
"terrain_interpretation",
|
|
["backend/app/services/flood_hazard_analysis_service.py"],
|
|
"deterministic_scenario_raster_analysis",
|
|
covered,
|
|
["depth MAE/RMSE", "hazard-class IoU", "coverage", "scenario identity"],
|
|
),
|
|
_capability(
|
|
"bathymetry_interpretation",
|
|
"terrain_interpretation",
|
|
[
|
|
"backend/app/services/bathymetry_raster_analysis_service.py",
|
|
"backend/app/services/mdk_bathymetry_probe_service.py",
|
|
],
|
|
"deterministic_vertical_reference_analysis",
|
|
covered,
|
|
["MAE", "RMSE", "bias", "coverage", "vertical-datum integrity"],
|
|
),
|
|
_capability(
|
|
"data_contract_validation_and_scan",
|
|
"geospatial_data_validation",
|
|
[
|
|
"backend/app/services/data_contract_validation.py",
|
|
"scripts/run_accuracy_phase3_full_data_scan.py",
|
|
],
|
|
"deterministic_validation",
|
|
synthetic,
|
|
["anomaly precision", "anomaly recall", "anomaly F1", "critical misses"],
|
|
),
|
|
_capability(
|
|
"aoi_partition_orchestration",
|
|
"geospatial_data_validation",
|
|
[
|
|
"backend/app/services/aoi_operation_service.py",
|
|
"backend/app/services/aoi_operation_executor.py",
|
|
"backend/app/services/aoi_operation_worker.py",
|
|
],
|
|
"deterministic_orchestration",
|
|
separate,
|
|
[
|
|
"partition completeness",
|
|
"overlap/gap",
|
|
"idempotency",
|
|
"resume correctness",
|
|
],
|
|
),
|
|
_capability(
|
|
"geo_assistant_orchestration",
|
|
"geospatial_data_validation",
|
|
["backend/app/services/geo_assistant_service.py"],
|
|
"tool_orchestration_interface",
|
|
"no_independent_accuracy_score_underlying_tool_results_are_authoritative",
|
|
["tool-selection correctness", "grounding", "unsupported-claim rate"],
|
|
),
|
|
]
|
|
|
|
|
|
def _numeric_metric(metrics: dict[str, Any], key: str) -> float | None:
|
|
value = metrics.get(key)
|
|
if isinstance(value, bool) or not isinstance(value, (int, float)):
|
|
return None
|
|
converted = float(value)
|
|
return converted if math.isfinite(converted) else None
|
|
|
|
|
|
def _macro_metrics(
|
|
values: list[dict[str, Any]],
|
|
keys: Iterable[str],
|
|
) -> dict[str, Any]:
|
|
output: dict[str, Any] = {
|
|
"status": "computed",
|
|
"method": "unweighted mean across cases with defined values",
|
|
}
|
|
computed = 0
|
|
for key in keys:
|
|
items = [
|
|
value
|
|
for metrics in values
|
|
if (value := _numeric_metric(metrics, key)) is not None
|
|
]
|
|
output[key] = mean(items) if items else None
|
|
output[f"{key}_case_support"] = len(items)
|
|
computed += bool(items)
|
|
if not computed:
|
|
output["status"] = "not_evaluable"
|
|
return output
|
|
|
|
|
|
def _aggregate_count_family(
|
|
values: list[dict[str, Any]],
|
|
extra_macro_keys: Iterable[str] = (),
|
|
) -> dict[str, Any]:
|
|
metrics = [item["metrics"] for item in values]
|
|
micro = count_metrics(
|
|
sum(int(item["true_positive"]) for item in metrics),
|
|
sum(int(item["false_positive"]) for item in metrics),
|
|
sum(int(item["false_negative"]) for item in metrics),
|
|
)
|
|
return {
|
|
"micro": micro,
|
|
"macro": _macro_metrics(
|
|
metrics,
|
|
("precision", "recall", "f1", *extra_macro_keys),
|
|
),
|
|
"observation_support": {
|
|
"references": sum(int(item["reference_count"]) for item in metrics),
|
|
"predictions": sum(int(item["prediction_count"]) for item in metrics),
|
|
},
|
|
"primary_metric": {
|
|
"name": "micro.f1",
|
|
"value": micro["f1"],
|
|
"direction": "higher_is_better",
|
|
},
|
|
}
|
|
|
|
|
|
def _aggregate_raster(values: list[dict[str, Any]]) -> dict[str, Any]:
|
|
metrics = [item["metrics"] for item in values]
|
|
class_counts: dict[str, dict[str, int]] = defaultdict(
|
|
lambda: {"tp": 0, "fp": 0, "fn": 0}
|
|
)
|
|
for item in metrics:
|
|
for class_name, class_metrics in item["per_class"].items():
|
|
class_counts[class_name]["tp"] += int(class_metrics["true_positive"])
|
|
class_counts[class_name]["fp"] += int(class_metrics["false_positive"])
|
|
class_counts[class_name]["fn"] += int(class_metrics["false_negative"])
|
|
per_class = {}
|
|
for class_name, counts in sorted(class_counts.items()):
|
|
class_metrics = count_metrics(counts["tp"], counts["fp"], counts["fn"])
|
|
class_metrics["iou"] = safe_rate(
|
|
counts["tp"],
|
|
counts["tp"] + counts["fp"] + counts["fn"],
|
|
)
|
|
per_class[class_name] = class_metrics
|
|
evaluated = sum(int(item["evaluated_reference_pixel_count"]) for item in metrics)
|
|
present = sum(int(item["prediction_present_pixel_count"]) for item in metrics)
|
|
correct = sum(int(item["correct_pixel_count"]) for item in metrics)
|
|
micro = {
|
|
"evaluated_reference_pixel_count": evaluated,
|
|
"prediction_present_pixel_count": present,
|
|
"correct_pixel_count": correct,
|
|
"accuracy": safe_rate(correct, evaluated),
|
|
"accuracy_ci95_wilson": wilson_interval(correct, evaluated),
|
|
"prediction_coverage": safe_rate(present, evaluated),
|
|
"prediction_coverage_ci95_wilson": wilson_interval(present, evaluated),
|
|
"mean_iou": _mean_or_none([item["iou"] for item in per_class.values()]),
|
|
"macro_f1_across_classes": _mean_or_none(
|
|
[item["f1"] for item in per_class.values()]
|
|
),
|
|
"per_class": per_class,
|
|
}
|
|
return {
|
|
"micro": micro,
|
|
"macro": _macro_metrics(
|
|
metrics,
|
|
("accuracy", "mean_iou", "macro_f1"),
|
|
),
|
|
"observation_support": {"evaluated_pixels": evaluated},
|
|
"primary_metric": {
|
|
"name": "micro.mean_iou",
|
|
"value": micro["mean_iou"],
|
|
"direction": "higher_is_better",
|
|
},
|
|
}
|
|
|
|
|
|
def _aggregate_terrain(values: list[dict[str, Any]]) -> dict[str, Any]:
|
|
metrics = [item["metrics"] for item in values]
|
|
units = sorted({str(item["units"]) for item in metrics})
|
|
references = sum(int(item["reference_count"]) for item in metrics)
|
|
evaluated = sum(int(item["evaluated_count"]) for item in metrics)
|
|
if len(units) != 1:
|
|
return {
|
|
"micro": {"status": "not_evaluable", "reason": "mixed units"},
|
|
"macro": {"status": "not_evaluable", "reason": "mixed units"},
|
|
"observation_support": {"references": references},
|
|
"primary_metric": {
|
|
"name": "micro.rmse",
|
|
"value": None,
|
|
"direction": "lower_is_better",
|
|
},
|
|
}
|
|
absolute_error_sum = sum(float(item["absolute_error_sum"]) for item in metrics)
|
|
squared_error_sum = sum(float(item["squared_error_sum"]) for item in metrics)
|
|
error_sum = sum(float(item["error_sum"]) for item in metrics)
|
|
micro = {
|
|
"status": "computed" if evaluated else "not_evaluable",
|
|
"units": units[0],
|
|
"reference_count": references,
|
|
"evaluated_count": evaluated,
|
|
"coverage": safe_rate(evaluated, references),
|
|
"coverage_ci95_wilson": wilson_interval(evaluated, references),
|
|
"mae": safe_rate(absolute_error_sum, evaluated),
|
|
"rmse": (math.sqrt(squared_error_sum / evaluated) if evaluated else None),
|
|
"bias": safe_rate(error_sum, evaluated),
|
|
}
|
|
return {
|
|
"micro": micro,
|
|
"macro": _macro_metrics(
|
|
metrics,
|
|
("coverage", "mae", "rmse", "bias"),
|
|
),
|
|
"observation_support": {
|
|
"references": references,
|
|
"evaluated": evaluated,
|
|
},
|
|
"primary_metric": {
|
|
"name": "micro.rmse",
|
|
"value": micro["rmse"],
|
|
"direction": "lower_is_better",
|
|
},
|
|
}
|
|
|
|
|
|
def _aggregate_task(values: list[dict[str, Any]]) -> dict[str, Any]:
|
|
task = values[0]["task"]
|
|
if task == "raster_classification":
|
|
aggregation = _aggregate_raster(values)
|
|
elif task == "terrain_interpretation":
|
|
aggregation = _aggregate_terrain(values)
|
|
else:
|
|
extras = {
|
|
"object_detection": ("ap50", "ap50_95"),
|
|
"footprint_segmentation": (
|
|
"mean_iou",
|
|
"mean_dice",
|
|
"mean_boundary_f1",
|
|
),
|
|
"vector_comparison": ("mean_iou",),
|
|
"change_detection": (),
|
|
"geospatial_data_validation": ("blocker_or_critical_miss_count",),
|
|
}[task]
|
|
aggregation = _aggregate_count_family(values, extras)
|
|
if task == "geospatial_data_validation":
|
|
aggregation["micro"]["blocker_or_critical_miss_count"] = sum(
|
|
int(item["metrics"]["blocker_or_critical_miss_count"])
|
|
for item in values
|
|
)
|
|
case_support = len(values)
|
|
status = (
|
|
"evaluable"
|
|
if case_support >= SUBGROUP_MIN_CASE_SUPPORT
|
|
and aggregation["primary_metric"]["value"] is not None
|
|
else "insufficient_support"
|
|
)
|
|
return {
|
|
"case_support": case_support,
|
|
"minimum_case_support": SUBGROUP_MIN_CASE_SUPPORT,
|
|
"status": status,
|
|
"release_gate_status": "not_evaluable",
|
|
"release_gate_reason": (
|
|
"insufficient_support"
|
|
if status == "insufficient_support"
|
|
else "no_frozen_release_target"
|
|
),
|
|
**aggregation,
|
|
}
|
|
|
|
|
|
def _stratum_key(value: Any) -> str:
|
|
return "__not_applicable__" if value is None else str(value)
|
|
|
|
|
|
def _worst_stratum_by_task(strata: dict[str, Any]) -> dict[str, Any]:
|
|
tasks = sorted(
|
|
{task for stratum in strata.values() for task in stratum["task_metrics"]}
|
|
)
|
|
output = {}
|
|
for task in tasks:
|
|
candidates = []
|
|
for stratum_name, stratum in strata.items():
|
|
task_metrics = stratum["task_metrics"].get(task)
|
|
if not task_metrics or task_metrics["status"] != "evaluable":
|
|
continue
|
|
primary = task_metrics["primary_metric"]
|
|
if primary["value"] is not None:
|
|
candidates.append((stratum_name, primary))
|
|
if not candidates:
|
|
output[task] = {
|
|
"status": "not_evaluable",
|
|
"reason": (
|
|
"no stratum meets minimum support with a defined primary metric"
|
|
),
|
|
}
|
|
continue
|
|
direction = candidates[0][1]["direction"]
|
|
selected = (
|
|
min(candidates, key=lambda item: (item[1]["value"], item[0]))
|
|
if direction == "higher_is_better"
|
|
else max(
|
|
candidates,
|
|
key=lambda item: (item[1]["value"], item[0]),
|
|
)
|
|
)
|
|
output[task] = {
|
|
"status": "computed",
|
|
"stratum": selected[0],
|
|
"primary_metric": selected[1],
|
|
}
|
|
return output
|
|
|
|
|
|
def subgroup_report(results: list[dict[str, Any]]) -> dict[str, Any]:
|
|
dimensions = (
|
|
"region",
|
|
"municipality",
|
|
"urbanity",
|
|
"object_size",
|
|
"source",
|
|
"sensor",
|
|
"resolution_m",
|
|
"season",
|
|
"date",
|
|
"vegetation",
|
|
"occlusion",
|
|
"difficulty",
|
|
)
|
|
dimension_reports = {}
|
|
any_insufficient = False
|
|
for dimension in dimensions:
|
|
groups: dict[str, list[dict[str, Any]]] = defaultdict(list)
|
|
for item in results:
|
|
groups[_stratum_key(item["metadata"].get(dimension))].append(item)
|
|
strata = {}
|
|
for key, values in sorted(groups.items()):
|
|
per_task: dict[str, list[dict[str, Any]]] = defaultdict(list)
|
|
for value in values:
|
|
per_task[value["task"]].append(value)
|
|
task_metrics = {
|
|
task: _aggregate_task(task_values)
|
|
for task, task_values in sorted(per_task.items())
|
|
}
|
|
insufficient = any(
|
|
item["status"] == "insufficient_support"
|
|
for item in task_metrics.values()
|
|
)
|
|
any_insufficient = any_insufficient or insufficient
|
|
strata[key] = {
|
|
"case_support": len(values),
|
|
"tasks": sorted(per_task),
|
|
"failure_count": sum(len(value["failures"]) for value in values),
|
|
"task_metrics": task_metrics,
|
|
"release_gate_status": "not_evaluable",
|
|
"release_gate_reason": "insufficient_support"
|
|
if insufficient
|
|
else "no_frozen_release_target",
|
|
}
|
|
dimension_reports[dimension] = {
|
|
"strata": strata,
|
|
"worst_stratum_by_task": _worst_stratum_by_task(strata),
|
|
}
|
|
return {
|
|
"schema_version": REPORT_SCHEMA_VERSION,
|
|
"minimum_case_support": SUBGROUP_MIN_CASE_SUPPORT,
|
|
"support_unit": (
|
|
"independent benchmark cases; pixel/object counts are reported "
|
|
"separately and do not replace case support"
|
|
),
|
|
"overall_status": (
|
|
"not_evaluable" if any_insufficient else "evaluable_no_release_target"
|
|
),
|
|
"dimensions": dimension_reports,
|
|
}
|
|
|
|
|
|
def evaluate_cases(path: Path, allowed_sample_ids: set[str]) -> dict[str, Any]:
|
|
file_hash = file_sha256(path)
|
|
portfolio = json.loads(path.read_text(encoding="utf-8"))
|
|
if not isinstance(portfolio, dict):
|
|
raise ValueError("The evaluation portfolio must be a JSON object")
|
|
claim_boundary = portfolio.get("claim_boundary")
|
|
if not isinstance(claim_boundary, str) or (
|
|
"synthetic" not in claim_boundary.lower()
|
|
):
|
|
raise ValueError(
|
|
"The portfolio claim boundary must explicitly state that it is synthetic"
|
|
)
|
|
split_roles = portfolio.get("split_roles")
|
|
if split_roles not in (["test"], ["test", "background-test"]):
|
|
raise ValueError(
|
|
"split_roles must be exactly ['test'] or ['test', 'background-test']"
|
|
)
|
|
if "challenge_cases" in portfolio or "challenge_labels" in portfolio:
|
|
raise ValueError("Challenge cases and labels must remain sealed and absent")
|
|
selection_policy = portfolio.get("selection_policy")
|
|
if not isinstance(selection_policy, str) or not selection_policy.strip():
|
|
raise ValueError("The portfolio requires an explicit selection policy")
|
|
declared_portfolio_lineage = _nonempty_mapping(
|
|
portfolio.get("portfolio_lineage"),
|
|
"portfolio_lineage",
|
|
)
|
|
for field in ("origin", "source_path", "version"):
|
|
value = declared_portfolio_lineage.get(field)
|
|
if not isinstance(value, str) or not value.strip():
|
|
raise ValueError(f"portfolio_lineage.{field} must be a non-empty string")
|
|
cases = portfolio.get("cases")
|
|
if not isinstance(cases, list) or not all(isinstance(item, dict) for item in cases):
|
|
raise ValueError("The portfolio cases must be an object list")
|
|
for case in cases:
|
|
if case.get("split") not in split_roles:
|
|
raise ValueError(
|
|
f"{case['sample_id']}: split must be one of {split_roles}; "
|
|
"challenge remains sealed"
|
|
)
|
|
_validate_case_common(case)
|
|
identifiers = [item["sample_id"] for item in cases]
|
|
duplicates = sorted(
|
|
{identifier for identifier in identifiers if identifiers.count(identifier) > 1}
|
|
)
|
|
if duplicates:
|
|
raise ValueError(f"Duplicate protected case ids: {duplicates}")
|
|
case_ids = set(identifiers)
|
|
unexpected = sorted(case_ids - allowed_sample_ids)
|
|
missing = sorted(allowed_sample_ids - case_ids)
|
|
if unexpected or missing:
|
|
raise ValueError(
|
|
"Protected case identity mismatch: "
|
|
f"unexpected={unexpected}, missing={missing}"
|
|
)
|
|
portfolio_canonical_hash = canonical_hash(portfolio)
|
|
results = [
|
|
EVALUATORS[item["task"]](item)
|
|
for item in sorted(cases, key=lambda item: item["sample_id"])
|
|
]
|
|
portfolio_lineage = {
|
|
"declared": serializable(declared_portfolio_lineage),
|
|
"portfolio_id": portfolio.get("portfolio_id"),
|
|
"portfolio_schema_version": portfolio.get("schema_version"),
|
|
"portfolio_file_sha256": file_hash,
|
|
"portfolio_canonical_json_sha256": portfolio_canonical_hash,
|
|
"source_path": declared_portfolio_lineage["source_path"],
|
|
"split_roles": list(split_roles),
|
|
"claim_boundary": claim_boundary,
|
|
}
|
|
for item in results:
|
|
item["raw"]["portfolio_lineage"] = portfolio_lineage
|
|
failures = sorted(
|
|
(failure for item in results for failure in item["failures"]),
|
|
key=lambda item: item["failure_id"],
|
|
)
|
|
results_hash = canonical_hash(results)
|
|
return {
|
|
"schema_version": REPORT_SCHEMA_VERSION,
|
|
"evaluator_version": EVALUATOR_VERSION,
|
|
"portfolio_id": portfolio["portfolio_id"],
|
|
"portfolio_file_sha256": file_hash,
|
|
"portfolio_canonical_json_sha256": portfolio_canonical_hash,
|
|
"claim_boundary": claim_boundary,
|
|
"selection_policy": selection_policy,
|
|
"split_roles": list(split_roles),
|
|
"protected_policy": serializable(portfolio.get("protected_policy")),
|
|
"hash_specification": {
|
|
"algorithm": "SHA-256",
|
|
"canonical_json": CANONICAL_JSON_SPEC,
|
|
"portfolio_file_sha256_input": "exact source-file bytes",
|
|
"portfolio_canonical_json_sha256_input": ("parsed complete portfolio"),
|
|
"results_canonical_json_sha256_input": (
|
|
"complete sample-id-ordered results array"
|
|
),
|
|
},
|
|
"task_inventory": task_inventory(),
|
|
"evaluated_task_families": sorted({item["task"] for item in results}),
|
|
"task_count": len({item["task"] for item in results}),
|
|
"case_count": len(results),
|
|
"results": results,
|
|
"subgroups": subgroup_report(results),
|
|
"failures": failures,
|
|
"results_canonical_json_sha256": results_hash,
|
|
}
|