fix(accuracy): close phase 4 evidence bypasses

backend/tests/test_accuracy_phase4_evaluator_hardening.py

@@ -17,6 +17,7 @@ if str(SCRIPTS) not in sys.path:
 
 from accuracy_phase4_evaluator import (  # noqa: E402
     TASKS,
+    EXPECTED_PROTECTED_POLICY,
     canonical_hash,
     count_metrics,
     detection_ap,
@@ -40,6 +41,7 @@ METADATA = {
     "source": "synthetic-source",
     "sensor": "synthetic-sensor",
     "resolution_m": 0.25,
+    "context": "dense_urban",
     "season": "summer",
     "date": "2026-01-01",
     "vegetation": "partial",
@@ -149,6 +151,7 @@ def test_raw_evidence_and_hashes_are_exact_and_recomputable(tmp_path: Path) -> N
     case = detection_case()
     portfolio = {
         "schema_version": 2,
+        "portfolio_kind": "synthetic_contract",
         "portfolio_id": "synthetic-hardening-test",
         "portfolio_lineage": {
             "origin": "repository_fixture",
@@ -158,7 +161,7 @@ def test_raw_evidence_and_hashes_are_exact_and_recomputable(tmp_path: Path) -> N
         "split_roles": ["test"],
         "selection_policy": "Fixed before evaluation; no selection.",
         "claim_boundary": "Synthetic evaluator test; not product accuracy.",
-        "protected_policy": {"threshold_selection_allowed": False},
+        "protected_policy": copy.deepcopy(EXPECTED_PROTECTED_POLICY),
         "cases": [case],
     }
     path = tmp_path / "portfolio.json"
@@ -203,6 +206,105 @@ def test_raw_evidence_and_hashes_are_exact_and_recomputable(tmp_path: Path) -> N
         evaluate_cases(path, {case["sample_id"]})
 
 
+def test_portfolio_schema_policy_metadata_and_lineage_are_strict(
+    tmp_path: Path,
+) -> None:
+    case = detection_case("strict-contract")
+    portfolio = {
+        "schema_version": 2,
+        "portfolio_kind": "synthetic_contract",
+        "portfolio_id": "synthetic-strict-contract",
+        "portfolio_lineage": {
+            "origin": "repository_fixture",
+            "source_path": "synthetic.json",
+            "version": "1",
+        },
+        "split_roles": ["test"],
+        "selection_policy": "Fixed before evaluation; no selection.",
+        "claim_boundary": "Synthetic evaluator test; not product accuracy.",
+        "protected_policy": copy.deepcopy(EXPECTED_PROTECTED_POLICY),
+        "cases": [case],
+    }
+    path = tmp_path / "strict.json"
+
+    def evaluate(value: dict) -> dict:
+        path.write_text(json.dumps(value, ensure_ascii=False), encoding="utf-8")
+        return evaluate_cases(path, {case["sample_id"]})
+
+    assert evaluate(portfolio)["case_count"] == 1
+
+    for invalid_version in (1, True, "2"):
+        invalid = copy.deepcopy(portfolio)
+        invalid["schema_version"] = invalid_version
+        with pytest.raises(
+            ValueError, match="schema_version must be exactly integer 2"
+        ):
+            evaluate(invalid)
+
+    invalid_policy = copy.deepcopy(portfolio)
+    invalid_policy["protected_policy"]["test_feedback_allowed"] = True
+    with pytest.raises(ValueError, match="protected_policy must exactly equal"):
+        evaluate(invalid_policy)
+
+    invalid_metadata = copy.deepcopy(portfolio)
+    invalid_metadata["cases"][0]["metadata"]["source"] = "unknown"
+    with pytest.raises(ValueError, match="metadata.source must be a meaningful"):
+        evaluate(invalid_metadata)
+
+    invalid_resolution = copy.deepcopy(portfolio)
+    invalid_resolution["cases"][0]["metadata"]["resolution_m"] = 0
+    with pytest.raises(ValueError, match="metadata.resolution_m must be positive"):
+        evaluate(invalid_resolution)
+
+    invalid_lineage = copy.deepcopy(portfolio)
+    del invalid_lineage["cases"][0]["lineage"]["prediction"]["derivation"]
+    with pytest.raises(ValueError, match="lineage.prediction missing"):
+        evaluate(invalid_lineage)
+
+
+def test_portfolio_kind_separates_synthetic_and_governed_product_claims(
+    tmp_path: Path,
+) -> None:
+    fixture_path = (
+        ROOT / "fixtures" / "accuracy" / "p4" / "protected-baseline-cases.json"
+    )
+    synthetic = json.loads(fixture_path.read_text(encoding="utf-8"))
+    allowed = {item["sample_id"] for item in synthetic["cases"]}
+    path = tmp_path / "portfolio.json"
+
+    missing_kind = copy.deepcopy(synthetic)
+    del missing_kind["portfolio_kind"]
+    path.write_text(json.dumps(missing_kind), encoding="utf-8")
+    with pytest.raises(ValueError, match="portfolio_kind"):
+        evaluate_cases(path, allowed)
+
+    confused = copy.deepcopy(synthetic)
+    confused["claim_boundary"] = "Governed product baseline accuracy evidence."
+    path.write_text(json.dumps(confused), encoding="utf-8")
+    with pytest.raises(ValueError, match="synthetic_contract"):
+        evaluate_cases(path, allowed)
+
+    governed = json.loads(
+        json.dumps(synthetic)
+        .replace("Synthetic", "Governed")
+        .replace("synthetic", "governed")
+        .replace("repository_fixture", "governed_product_evaluation")
+    )
+    governed["portfolio_kind"] = "governed_product_baseline"
+    governed["portfolio_id"] = "governed-product-baseline-test"
+    governed["claim_boundary"] = (
+        "Governed product baseline metrics recomputed from protected raw cases; "
+        "inference provenance is validated separately."
+    )
+    path.write_text(json.dumps(governed), encoding="utf-8")
+    report = evaluate_cases(path, allowed)
+    assert report["portfolio_kind"] == "governed_product_baseline"
+    assert set(report["evaluated_task_families"]) == TASKS
+
+    governed["cases"] = governed["cases"][:-1]
+    path.write_text(json.dumps(governed), encoding="utf-8")
+
+
 def test_ap_ties_use_stable_ids_and_matching_is_class_aware() -> None:
     references = [{"id": "r", "class": "building", "bbox": [0, 0, 4, 4]}]
     predictions = [
@@ -233,6 +335,83 @@ def test_ap_ties_use_stable_ids_and_matching_is_class_aware() -> None:
     assert detection_ap(wrong_class, references, 0.5) == pytest.approx(0.5)
 
 
+def test_detection_ap_and_calibration_are_pooled_globally_and_per_subgroup(
+    tmp_path: Path,
+) -> None:
+    first = detection_case("a-case")
+    first["predictions"] = [
+        {
+            "id": "p-true",
+            "class": "building",
+            "bbox": [0, 0, 4, 4],
+            "confidence": 0.9,
+        }
+    ]
+    second = detection_case("b-case")
+    second["predictions"] = [
+        {
+            "id": "p-false",
+            "class": "building",
+            "bbox": [10, 10, 12, 12],
+            "confidence": 0.9,
+        },
+        {
+            "id": "p-true",
+            "class": "building",
+            "bbox": [0, 0, 4, 4],
+            "confidence": 0.8,
+        },
+    ]
+    portfolio = {
+        "schema_version": 2,
+        "portfolio_kind": "synthetic_contract",
+        "portfolio_id": "synthetic-pooled-detection",
+        "portfolio_lineage": {
+            "origin": "repository_fixture",
+            "source_path": "pooled.json",
+            "version": "1",
+        },
+        "split_roles": ["test"],
+        "selection_policy": "Fixed before evaluation; no selection.",
+        "claim_boundary": "Synthetic evaluator test; not product accuracy.",
+        "protected_policy": copy.deepcopy(EXPECTED_PROTECTED_POLICY),
+        "cases": [first, second],
+    }
+    path = tmp_path / "pooled.json"
+    path.write_text(json.dumps(portfolio, ensure_ascii=False), encoding="utf-8")
+    report = evaluate_cases(path, {"a-case", "b-case"})
+
+    case_ap = [item["metrics"]["ap50"] for item in report["results"]]
+    pooled = report["portfolio_metrics"]["object_detection"]["micro"]
+    expected_pooled = detection_ap(
+        [
+            {**item, "id": f"a-case::{item['id']}", "_sample_id": "a-case"}
+            for item in first["predictions"]
+        ]
+        + [
+            {**item, "id": f"b-case::{item['id']}", "_sample_id": "b-case"}
+            for item in second["predictions"]
+        ],
+        [
+            {**item, "id": f"a-case::{item['id']}", "_sample_id": "a-case"}
+            for item in first["references"]
+        ]
+        + [
+            {**item, "id": f"b-case::{item['id']}", "_sample_id": "b-case"}
+            for item in second["references"]
+        ],
+        0.5,
+    )
+    assert pooled["ap50"] == expected_pooled
+    assert pooled["ap50"] != pytest.approx(sum(case_ap) / len(case_ap))
+    assert sum(item["count"] for item in pooled["calibration"]["bins"]) == 3
+    subgroup = report["subgroups"]["dimensions"]["region"]["strata"]["flanders"]
+    subgroup_calibration = subgroup["task_metrics"]["object_detection"]["micro"][
+        "calibration"
+    ]
+    assert sum(item["count"] for item in subgroup_calibration["bins"]) == 3
+
+
 def test_raster_requires_exact_rectangular_alignment_masks_nodata_and_classes() -> None:
     invalid_case = detection_case("invalid-class")
     invalid_case["predictions"][0]["class"] = "road"
@@ -270,6 +449,11 @@ def test_raster_requires_exact_rectangular_alignment_masks_nodata_and_classes()
     invalid_nodata["predictions"][0][0] = -9999
     with pytest.raises(ValueError, match="marks nodata as valid"):
         evaluate_raster_classification(invalid_nodata)
+    singular = raster_case("singular")
+    for side in ("reference", "prediction"):
+        singular["raster_context"][side]["transform"] = [1, 2, 0, 2, 4, 0]
+    with pytest.raises(ValueError, match="affine transform is singular"):
+        evaluate_raster_classification(singular)
 
 
 def test_polygon_metrics_require_valid_geometry_projected_crs_and_metres() -> None:
@@ -280,11 +464,59 @@ def test_polygon_metrics_require_valid_geometry_projected_crs_and_metres() -> No
         ]["mean_iou"]
         == 1.0
     )
+    outer = [
+        [100000, 200000],
+        [100020, 200000],
+        [100020, 200020],
+        [100000, 200020],
+        [100000, 200000],
+    ]
+    hole = [
+        [100005, 200005],
+        [100010, 200005],
+        [100010, 200010],
+        [100005, 200010],
+        [100005, 200005],
+    ]
+    polygon_geometry = {"type": "Polygon", "coordinates": [outer, hole]}
+    geojson_polygon = polygon_case()
+    for side in ("references", "predictions"):
+        del geojson_polygon[side][0]["polygon"]
+        geojson_polygon[side][0]["geometry"] = copy.deepcopy(polygon_geometry)
+    polygon_result = evaluate_vector_comparison(geojson_polygon)
+    assert polygon_result["metrics"]["mean_iou"] == 1.0
+    assert polygon_result["raw"]["references"][0]["geometry"] == polygon_geometry
+
+    second = [
+        [100030, 200000],
+        [100040, 200000],
+        [100040, 200010],
+        [100030, 200010],
+        [100030, 200000],
+    ]
+    multipolygon_geometry = {
+        "type": "MultiPolygon",
+        "coordinates": [[outer, hole], [second]],
+    }
+    geojson_multi = polygon_case()
+    for side in ("references", "predictions"):
+        del geojson_multi[side][0]["polygon"]
+        geojson_multi[side][0]["geometry"] = copy.deepcopy(multipolygon_geometry)
+    assert evaluate_vector_comparison(geojson_multi)["metrics"]["mean_iou"] == 1.0
 
     geographic = polygon_case()
     geographic["spatial_context"]["crs"] = "EPSG:4326"
     with pytest.raises(ValueError, match="projected CRS"):
         evaluate_vector_comparison(geographic)
+    mercator = polygon_case()
+    mercator["spatial_context"]["crs"] = "EPSG:3857"
+    with pytest.raises(ValueError, match="Mercator is unsuitable"):
+        evaluate_vector_comparison(mercator)
+
+    wrong_geography = polygon_case()
+    wrong_geography["spatial_context"]["crs"] = "EPSG:32660"
+    with pytest.raises(ValueError, match="does not overlap"):
+        evaluate_vector_comparison(wrong_geography)
 
     wrong_units = polygon_case()
     wrong_units["spatial_context"]["coordinate_units"] = "degree"
@@ -303,6 +535,61 @@ def test_polygon_metrics_require_valid_geometry_projected_crs_and_metres() -> No
         evaluate_vector_comparison(bowtie)
 
 
+def test_failure_gallery_covers_geometry_raster_calibration_and_contexts() -> None:
+    segmentation = polygon_case("footprint_segmentation")
+    segmentation["predictions"][0]["polygon"] = [
+        [100000, 200000],
+        [100012, 200000],
+        [100012, 200010],
+        [100000, 200010],
+        [100000, 200000],
+    ]
+    segmentation_result = evaluate_footprint_segmentation(segmentation)
+    segmentation_codes = {
+        item["error_code"] for item in segmentation_result["failures"]
+    }
+    assert {"M-BOUNDARY", "M-AREA-BIAS"} <= segmentation_codes
+
+    raster = raster_case("raster-taxonomy")
+    raster["metadata"]["tile_edge"] = True
+    raster["predictions"][0][0] = 1
+    raster_result = evaluate_raster_classification(raster)
+    raster_failure = next(
+        item
+        for item in raster_result["failures"]
+        if item["kind"] == "raster_misclassification"
+    )
+    assert raster_failure["error_code"] == "M-CLASS"
+    assert "tile_edge" in raster_failure["contexts"]
+
+    detection = detection_case("context-taxonomy")
+    detection["references"] = []
+    detection["predictions"] = [
+        {
+            "id": "high-confidence-fp",
+            "class": "building",
+            "bbox": [10, 10, 12, 12],
+            "confidence": 0.95,
+        }
+    ]
+    detection["config"]["fixed_diagnostic_risk_thresholds"] = [0.5, 0.9]
+    detection["metadata"]["tile_edge"] = True
+    detection["metadata"]["ood"] = True
+    detection_result = evaluate_object_detection(detection)
+    false_positive = next(
+        item
+        for item in detection_result["failures"]
+        if item["kind"] == "false_positive"
+    )
+    assert {"tile_edge", "high_confidence", "out_of_distribution"} <= set(
+        false_positive["contexts"]
+    )
+    assert {"M-MISCALIBRATED", "M-OOD"} <= set(false_positive["secondary_error_codes"])
+    assert any(
+        item["error_code"] == "M-MISCALIBRATED" for item in detection_result["failures"]
+    )
+
+
 def test_terrain_rejects_non_finite_and_validation_counts_only_critical_misses() -> (
     None
 ):
@@ -345,26 +632,60 @@ def test_terrain_rejects_non_finite_and_validation_counts_only_critical_misses()
         evaluate_validation(validation)["metrics"]["blocker_or_critical_miss_count"]
         == 1
     )
+    validation["expected_anomalies"] = [{"code": "D-SEVERITY", "severity": "critical"}]
+    validation["observed_anomalies"] = [{"code": "D-SEVERITY", "severity": "minor"}]
+    severity_result = evaluate_validation(validation)
+    assert severity_result["metrics"]["true_positive"] == 0
+    assert severity_result["metrics"]["false_positive"] == 1
+    assert severity_result["metrics"]["false_negative"] == 1
+    assert severity_result["metrics"]["severity_mismatch_count"] == 1
+    assert severity_result["metrics"]["blocker_or_critical_miss_count"] == 1
+
     validation["expected_anomalies"] = ["D-NO-SEVERITY"]
     with pytest.raises(ValueError, match="include code and severity"):
         evaluate_validation(validation)
 
 
-def _subgroup_result(region: str, tp: int, fp: int, fn: int) -> dict:
+def _subgroup_result(region: str, index: int, tp: int, fp: int, fn: int) -> dict:
     metadata = copy.deepcopy(METADATA)
     metadata["region"] = region
+    sample_id = f"{region}-{index}"
+    reference = {"id": "r", "class": "building", "bbox": [0, 0, 1, 1]}
+    prediction = {
+        "id": "p",
+        "class": "building",
+        "bbox": [0, 0, 1, 1],
+        "confidence": 0.8,
+    }
     return {
+        "sample_id": sample_id,
         "task": "object_detection",
         "metadata": metadata,
         "metrics": {**count_metrics(tp, fp, fn), "ap50": 0.5, "ap50_95": 0.4},
+        "raw": {
+            "sample_id": sample_id,
+            "classes": ["building"],
+            "references": [reference],
+            "predictions_pre_filter": [prediction],
+            "predictions_post_filter": [prediction],
+            "matches": [
+                {
+                    "prediction_id": "p",
+                    "reference_id": "r",
+                    "overlap": 1.0,
+                    "confidence": 0.8,
+                    "class": "building",
+                }
+            ],
+        },
         "failures": [],
     }
 
 
 def test_subgroups_report_task_metrics_support_ci_and_worst_stratum() -> None:
     results = [
-        *[_subgroup_result("strong", 10, 0, 0) for _ in range(5)],
+        *[_subgroup_result("strong", index, 10, 0, 0) for index in range(5)],
-        *[_subgroup_result("weak", 1, 4, 4) for _ in range(5)],
+        *[_subgroup_result("weak", index, 1, 4, 4) for index in range(5)],
     ]
     report = subgroup_report(results)
     region = report["dimensions"]["region"]
@@ -376,7 +697,7 @@ def test_subgroups_report_task_metrics_support_ci_and_worst_stratum() -> None:
     assert weak["macro"]["f1_case_support"] == 5
     assert region["worst_stratum_by_task"]["object_detection"]["stratum"] == "weak"
 
-    insufficient = subgroup_report([_subgroup_result("thin", 1, 0, 0)])
+    insufficient = subgroup_report([_subgroup_result("thin", 0, 1, 0, 0)])
     thin = insufficient["dimensions"]["region"]["strata"]["thin"]
     assert thin["task_metrics"]["object_detection"]["status"] == "insufficient_support"
     assert thin["release_gate_status"] == "not_evaluable"

backend/tests/test_accuracy_phase4_split_hardening.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import copy
+import hashlib
 import json
 import sys
 from pathlib import Path
@@ -28,13 +29,25 @@ def load_source() -> dict:
     return json.loads(SOURCE.read_text(encoding="utf-8"))
 
 
+def build_fixture_manifests(source: dict) -> tuple[dict, dict, dict]:
+    return build_manifests(source, trusted_fixture_mode=True)
+
+
+def assert_fixture_training_inputs_safe(
+    input_paths: list[Path], input_records: list[dict], protected: dict
+) -> None:
+    assert_training_inputs_safe(
+        input_paths, input_records, protected, trusted_fixture_mode=True
+    )
+
+
 def test_normative_roles_hashes_and_source_order_are_enforced() -> None:
     source = load_source()
-    development, protected, leakage = build_manifests(source)
+    development, protected, leakage = build_fixture_manifests(source)
     reversed_source = copy.deepcopy(source)
     reversed_source["samples"].reverse()
-    reversed_development, reversed_protected, reversed_leakage = build_manifests(
+    reversed_development, reversed_protected, reversed_leakage = (
-        reversed_source
+        build_fixture_manifests(reversed_source)
     )
 
     assert leakage["status"] == "pass"
@@ -82,7 +95,7 @@ def test_cross_split_lineage_and_content_collisions_fail(
 ) -> None:
     source = load_source()
     source["samples"][8][field] = source["samples"][0][field]
-    _development, _protected, leakage = build_manifests(source)
+    _development, _protected, leakage = build_fixture_manifests(source)
 
     assert leakage["status"] == "fail"
     assert expected_code in {item["code"] for item in leakage["findings"]}
@@ -98,7 +111,7 @@ def test_cross_split_lineage_and_content_collisions_fail(
 def test_near_duplicate_fingerprints_fail(field: str, expected_code: str) -> None:
     source = load_source()
     source["samples"][8][field] = source["samples"][0][field]
-    _development, _protected, leakage = build_manifests(source)
+    _development, _protected, leakage = build_fixture_manifests(source)
 
     assert leakage["status"] == "fail"
     assert expected_code in {item["code"] for item in leakage["findings"]}
@@ -111,7 +124,7 @@ def test_object_native_feature_and_spatial_collisions_fail() -> None:
         "native_feature_ids"
     ]
     source["samples"][10]["bbox"] = source["samples"][2]["bbox"]
-    _development, _protected, leakage = build_manifests(source)
+    _development, _protected, leakage = build_fixture_manifests(source)
     codes = {item["code"] for item in leakage["findings"]}
 
     assert {"S-OBJECT-INSTANCE", "S-NATIVE-FEATURE", "S-SPATIAL-OVERLAP"} <= codes
@@ -121,32 +134,34 @@ def test_non_metric_crs_and_missing_normative_role_fail_closed() -> None:
     geographic = load_source()
     geographic["crs"] = "EPSG:4326"
     with pytest.raises(LeakageError, match="projected in metres"):
-        build_manifests(geographic)
+        build_fixture_manifests(geographic)
 
     missing = load_source()
     missing["samples"] = [
         item for item in missing["samples"] if item["split"] != "calibration"
     ]
     with pytest.raises(LeakageError, match="Required splits are absent"):
-        build_manifests(missing)
+        build_fixture_manifests(missing)
 
 
 def test_training_firewall_only_allows_train_and_binds_protected_lineage() -> None:
-    development, protected, leakage = build_manifests(load_source())
+    development, protected, leakage = build_fixture_manifests(load_source())
     assert leakage["status"] == "pass"
     train = [item for item in development["samples"] if item["split"] == "train"]
     validation = next(item for item in development["samples"] if item["split"] == "val")
     protected_item = protected["samples"][0]
 
-    assert_training_inputs_safe([], train, protected)
+    assert_fixture_training_inputs_safe([], train, protected)
     with pytest.raises(LeakageError, match="non_train_role"):
-        assert_training_inputs_safe([], [validation], protected)
+        assert_fixture_training_inputs_safe([], [validation], protected)
     with pytest.raises(LeakageError, match="protected_identity"):
         disguised = copy.deepcopy(train[0])
         disguised["source_family"] = protected_item["source_family"]
-        assert_training_inputs_safe([], [disguised], protected)
+        assert_fixture_training_inputs_safe([], [disguised], protected)
     with pytest.raises(LeakageError, match="protected_path"):
-        assert_training_inputs_safe([Path("vault/protected/test.json")], [], protected)
+        assert_fixture_training_inputs_safe(
+            [Path("vault/protected/test.json")], [], protected
+        )
 
 
 def test_failed_generation_writes_status_but_no_consumable_manifests(
@@ -159,12 +174,18 @@ def test_failed_generation_writes_status_but_no_consumable_manifests(
     output = tmp_path / "out"
 
     with pytest.raises(LeakageError, match="Leakage gate failed"):
-        generate(source_path, output)
+        generate(source_path, output, trusted_fixture_mode=True)
 
     status = json.loads((output / "generation-status.json").read_text(encoding="utf-8"))
     assert status["status"] == "fail"
-    assert not (output / "development-split-manifest.json").exists()
+    assert status["consumable_manifests_valid"] is False
-    assert not (output / "protected-split-manifest.json").exists()
+    for name in (
+        "development-split-manifest.json",
+        "protected-split-manifest.json",
+    ):
+        tombstone = json.loads((output / name).read_text(encoding="utf-8"))
+        assert tombstone["status"] == "invalidated"
+        assert tombstone["consumable"] is False
 
 
 def test_deterministic_grouped_assignment_is_stable_and_keeps_relatives_together() -> (
@@ -196,11 +217,11 @@ def test_deterministic_grouped_assignment_is_stable_and_keeps_relatives_together
         item.pop("split")
     source["samples"][1]["group_id"] = source["samples"][0]["group_id"]
 
-    development, protected, leakage = build_manifests(source)
+    development, protected, leakage = build_fixture_manifests(source)
     reversed_source = copy.deepcopy(source)
     reversed_source["samples"].reverse()
-    reversed_development, reversed_protected, reversed_leakage = build_manifests(
+    reversed_development, reversed_protected, reversed_leakage = (
-        reversed_source
+        build_fixture_manifests(reversed_source)
     )
 
     assigned = {
@@ -220,3 +241,298 @@ def test_deterministic_grouped_assignment_is_stable_and_keeps_relatives_together
     assert reversed_development["manifest_sha256"] == development["manifest_sha256"]
     assert reversed_protected["manifest_sha256"] == protected["manifest_sha256"]
     assert reversed_leakage == leakage
+
+
+def test_source_cannot_weaken_mandatory_roles_or_policy_floors() -> None:
+    source = load_source()
+    source["required_splits"] = ["train", "val", "test"]
+    with pytest.raises(LeakageError, match="mandatory role order"):
+        build_fixture_manifests(source)
+
+    grouped = load_source()
+    grouped["assignment_mode"] = "deterministic_grouped"
+    grouped["split_assignment"] = {"roles": ["train", "val"]}
+    for item in grouped["samples"]:
+        item.pop("split")
+    with pytest.raises(LeakageError, match="mandatory role order"):
+        build_fixture_manifests(grouped)
+
+    for field, value in (
+        ("independence_buffer_m", 1999),
+        ("perceptual_hamming_threshold", 3),
+        ("label_geometry_hamming_threshold", 1),
+    ):
+        weakened = load_source()
+        weakened[field] = value
+        with pytest.raises(LeakageError, match="code-owned minimum"):
+            build_fixture_manifests(weakened)
+
+
+def test_task_coverage_gap_and_even_justified_exemption_fail_honestly() -> None:
+    source = load_source()
+    source["samples"] = [
+        item
+        for item in source["samples"]
+        if item["sample_id"] != "validation-test-national"
+    ]
+    _development, _protected, leakage = build_fixture_manifests(source)
+    assert leakage["status"] == "fail"
+    assert "S-PROTECTED-TASK-COVERAGE-MISSING" in {
+        finding["code"] for finding in leakage["findings"]
+    }
+
+    source["protected_task_exemptions"] = {
+        "geospatial_data_validation": (
+            "No evaluator-visible reference exists; challenge data remains sealed."
+        )
+    }
+    _development, _protected, leakage = build_fixture_manifests(source)
+    assert leakage["status"] == "fail"
+    assert "S-PROTECTED-TASK-COVERAGE-EXEMPTED" in {
+        finding["code"] for finding in leakage["findings"]
+    }
+
+
+def test_identifiers_and_acquisition_dates_are_canonical_leakage_keys() -> None:
+    source = load_source()
+    source["samples"][8]["group_id"] = "  G01  "
+    _development, _protected, leakage = build_fixture_manifests(source)
+    assert "S-SPATIAL-GROUP" in {item["code"] for item in leakage["findings"]}
+
+    temporal = load_source()
+    temporal["samples"][8]["acquisition_date"] = temporal["samples"][0][
+        "acquisition_date"
+    ]
+    _development, _protected, leakage = build_fixture_manifests(temporal)
+    assert "S-ACQUISITION-DATE" in {item["code"] for item in leakage["findings"]}
+
+    ambiguous = load_source()
+    ambiguous["samples"][8]["sample_id"] = "  DET-TRAIN-A  "
+    with pytest.raises(LeakageError, match="ambiguous canonical sample_id"):
+        build_fixture_manifests(ambiguous)
+
+
+def test_challenge_is_sealed_in_standard_manifest() -> None:
+    _development, protected, leakage = build_fixture_manifests(load_source())
+    assert leakage["status"] == "pass"
+    challenge = [item for item in protected["samples"] if item["split"] == "challenge"]
+    forbidden = {
+        "label_sha256",
+        "label_geometry_hash",
+        "label_geometry_fingerprint",
+        "object_ids",
+        "native_feature_ids",
+        "record_sha256",
+        "label_path",
+        "label_geometry_path",
+    }
+    assert challenge
+    assert all(item["sealed"] is True for item in challenge)
+    assert all(not (forbidden & set(item)) for item in challenge)
+
+
+def test_firewall_rejects_empty_tampered_wrong_and_renamed_manifests(
+    tmp_path: Path,
+) -> None:
+    development, protected, leakage = build_fixture_manifests(load_source())
+    assert leakage["status"] == "pass"
+    train = [item for item in development["samples"] if item["split"] == "train"]
+
+    with pytest.raises(LeakageError, match="empty manifest"):
+        assert_fixture_training_inputs_safe([], train, {})
+    with pytest.raises(LeakageError, match="missing fields"):
+        assert_fixture_training_inputs_safe([], train, development)
+    tampered = copy.deepcopy(protected)
+    tampered["samples"].pop()
+    with pytest.raises(LeakageError, match="checksum mismatch"):
+        assert_fixture_training_inputs_safe([], train, tampered)
+
+    renamed = tmp_path / "ordinary-training-input.json"
+    renamed.write_text(json.dumps(protected), encoding="utf-8")
+    with pytest.raises(LeakageError, match="protected_manifest_content"):
+        assert_fixture_training_inputs_safe([renamed], train, protected)
+
+
+def _canonical_json_sha256(value: object) -> str:
+    return hashlib.sha256(
+        json.dumps(
+            value, ensure_ascii=False, sort_keys=True, separators=(",", ":")
+        ).encode("utf-8")
+    ).hexdigest()
+
+
+def make_governed_source(tmp_path: Path) -> dict:
+    source = load_source()
+    source["dataset_version"] = "governed-production-v1"
+    source["claim_boundary"] = "Governed production split source."
+    p3_items: list[dict] = []
+    provenance_records: list[dict] = []
+    asset_fields = {
+        "raw_image": ("raw_image_path", "raw_image_sha256"),
+        "processed_image": ("processed_image_path", "processed_image_sha256"),
+        "label": ("label_path", "label_sha256"),
+        "label_geometry": ("label_geometry_path", "label_geometry_hash"),
+    }
+    for sample in source["samples"]:
+        assets: dict[str, dict] = {}
+        p3_ids: dict[str, str] = {}
+        for role, (path_field, hash_field) in asset_fields.items():
+            relative = Path("assets") / sample["sample_id"] / f"{role}.bin"
+            path = tmp_path / relative
+            path.parent.mkdir(parents=True, exist_ok=True)
+            path.write_bytes(f"{sample['sample_id']}:{role}:governed".encode())
+            digest = hashlib.sha256(path.read_bytes()).hexdigest()
+            p3_id = hashlib.sha256(
+                f"{sample['sample_id']}:{role}".encode()
+            ).hexdigest()[:20]
+            relative_posix = relative.as_posix()
+            sample[path_field] = relative_posix
+            sample[hash_field] = digest
+            p3_ids[role] = p3_id
+            assets[role] = {
+                "path": relative_posix,
+                "sha256": digest,
+                "size_bytes": path.stat().st_size,
+                "p3_item_id": p3_id,
+            }
+            p3_items.append(
+                {
+                    "item_id": p3_id,
+                    "path": relative_posix,
+                    "sha256": digest,
+                    "size_bytes": path.stat().st_size,
+                    "status": "examined",
+                    "read_status": "readable",
+                    "recommended_action": "accept",
+                    "empty_content": False,
+                    "schema_conformity": "conformant",
+                    "anomalies": [],
+                }
+            )
+        provenance_id = (
+            "prov-" + hashlib.sha256(sample["sample_id"].encode()).hexdigest()[:24]
+        )
+        sample["governance_binding"] = {
+            "source_provenance_record_id": provenance_id,
+            "p3_item_ids": p3_ids,
+        }
+        provenance_records.append(
+            {
+                "record_id": provenance_id,
+                "sample_id": sample["sample_id"],
+                "status": "accepted",
+                "lineage_status": "complete",
+                "training_allowed": True,
+                "perceptual_image_hash": sample["perceptual_image_hash"],
+                "label_geometry_fingerprint": sample["label_geometry_fingerprint"],
+                "assets": assets,
+            }
+        )
+    p3 = {
+        "schema_version": 1,
+        "scan_id": "p3-test-governed",
+        "scanner_version": "3.0.3",
+        "completed_at": "2026-08-02T12:00:00+02:00",
+        "items": p3_items,
+        "reconciliation": {
+            "examined": len(p3_items),
+            "skipped": 0,
+            "unreachable": 0,
+            "inventory_total": len(p3_items),
+            "reconciles": True,
+        },
+    }
+    provenance = {
+        "schema_version": 1,
+        "manifest_type": "geointel_phase4_source_provenance",
+        "status": "pass",
+        "records": provenance_records,
+        "records_canonical_json_sha256": _canonical_json_sha256(provenance_records),
+    }
+    p3_path = tmp_path / "p3.json"
+    provenance_path = tmp_path / "provenance.json"
+    p3_path.write_text(json.dumps(p3), encoding="utf-8")
+    provenance_path.write_text(json.dumps(provenance), encoding="utf-8")
+    source["governance_evidence"] = {
+        "p3_scan_manifest": {
+            "path": p3_path.name,
+            "sha256": hashlib.sha256(p3_path.read_bytes()).hexdigest(),
+        },
+        "source_provenance_manifest": {
+            "path": provenance_path.name,
+            "sha256": hashlib.sha256(provenance_path.read_bytes()).hexdigest(),
+        },
+    }
+    return source
+
+
+def test_fixture_mode_is_explicit_and_source_metadata_cannot_enable_it() -> None:
+    with pytest.raises(LeakageError, match="explicit trusted_fixture_mode"):
+        build_manifests(load_source())
+
+    source = load_source()
+    source["trusted_fixture_mode"] = True
+    with pytest.raises(LeakageError, match="cannot be enabled by source metadata"):
+        build_manifests(source, trusted_fixture_mode=True)
+
+
+def test_empty_or_arbitrary_governance_json_is_rejected(tmp_path: Path) -> None:
+    source = load_source()
+    source["dataset_version"] = "governed-production-v1"
+    source["claim_boundary"] = "Governed production split source."
+    p3 = tmp_path / "p3.json"
+    provenance = tmp_path / "provenance.json"
+    p3.write_text("{}", encoding="utf-8")
+    provenance.write_text("{}", encoding="utf-8")
+    source["governance_evidence"] = {
+        "p3_scan_manifest": {
+            "path": p3.name,
+            "sha256": hashlib.sha256(p3.read_bytes()).hexdigest(),
+        },
+        "source_provenance_manifest": {
+            "path": provenance.name,
+            "sha256": hashlib.sha256(provenance.read_bytes()).hexdigest(),
+        },
+    }
+    with pytest.raises(LeakageError, match="non-empty JSON object"):
+        build_manifests(source, source_root=tmp_path)
+
+
+def test_governed_records_require_exact_provenance_paths_and_live_bytes(
+    tmp_path: Path,
+) -> None:
+    source = make_governed_source(tmp_path)
+    development, protected, leakage = build_manifests(source, source_root=tmp_path)
+    assert leakage["status"] == "pass"
+    assert protected["source_trust"]["production_accuracy_use_allowed"] is True
+    train = [item for item in development["samples"] if item["split"] == "train"]
+    assert_training_inputs_safe([], train, protected)
+
+    no_paths = copy.deepcopy(train[0])
+    no_paths.pop("content_path_bindings")
+    with pytest.raises(LeakageError, match="missing_accessible_content_paths"):
+        assert_training_inputs_safe([], [no_paths], protected)
+
+    relabeled = copy.deepcopy(
+        next(item for item in protected["samples"] if item["split"] == "test")
+    )
+    relabeled["split"] = "train"
+    relabeled.pop("content_path_bindings")
+    for field in ("sample_id", "group_id", "source_family", "temporal_family"):
+        relabeled[field] = f"spoofed-{field}"
+    with pytest.raises(
+        LeakageError, match="unavailable_provenance_record|protected_identity"
+    ):
+        assert_training_inputs_safe([], [relabeled], protected)
+
+    broken_binding = copy.deepcopy(source)
+    broken_binding["samples"][0]["governance_binding"]["p3_item_ids"]["raw_image"] = (
+        "0" * 20
+    )
+    with pytest.raises(LeakageError, match="provenance binding mismatch|P3 record"):
+        build_manifests(broken_binding, source_root=tmp_path)
+
+    raw_path = Path(train[0]["content_path_bindings"]["raw_image"]["resolved_path"])
+    raw_path.write_bytes(b"mutated after manifest creation")
+    with pytest.raises(LeakageError, match="record_path_hash_binding_mismatch"):
+        assert_training_inputs_safe([], train, protected)

fixtures/accuracy/p4/protected-baseline-cases.json

@@ -1,5 +1,6 @@
 {
   "schema_version": 2,
+  "portfolio_kind": "synthetic_contract",
   "portfolio_id": "geointel-p4-reference-harness-v2",
   "split_roles": [
     "test",

scripts/accuracy_phase4_evaluator.py

@@ -1,7 +1,8 @@
 """Fail-closed, task-aware metrics for the GeoIntel Phase 4 benchmark.
 
-The bundled portfolio is a synthetic contract fixture. Nothing emitted by
+The evaluator accepts explicitly separated synthetic contract portfolios and
-this module is, by itself, evidence of production-model accuracy.
+governed product-baseline portfolios. Nothing emitted by this module is, by
+itself, evidence that inference provenance or production release gates passed.
 """
 
 from __future__ import annotations
@@ -14,7 +15,8 @@ from pathlib import Path
 from statistics import mean, median
 from typing import Any, Callable, Iterable
 
-EVALUATOR_VERSION = "2.0.0"
+PORTFOLIO_KINDS = {"synthetic_contract", "governed_product_baseline"}
+EVALUATOR_VERSION = "2.1.0"
 REPORT_SCHEMA_VERSION = 2
 SUBGROUP_MIN_CASE_SUPPORT = 5
 CANONICAL_JSON_SPEC = (
@@ -46,6 +48,54 @@ ERROR_CODES = {
     "validation_false_positive": "D-VALIDATION-FP",
     "validation_false_negative": "D-VALIDATION-FN",
     "terrain_missing": "P-PARTIAL",
+    "raster_misclassification": "M-CLASS",
+    "raster_missing": "P-PARTIAL",
+    "boundary_error": "M-BOUNDARY",
+    "area_bias": "M-AREA-BIAS",
+    "miscalibrated": "M-MISCALIBRATED",
+}
+EXPECTED_PROTECTED_POLICY = {
+    "operating_point_selection_allowed": False,
+    "diagnostic_curves_select_operating_point": False,
+    "test_feedback_allowed": False,
+    "threshold_selection_source": "pre_registered_configuration_only",
+}
+REQUIRED_METADATA_STRING_FIELDS = (
+    "region",
+    "municipality",
+    "urbanity",
+    "object_size",
+    "source",
+    "sensor",
+    "season",
+    "date",
+    "vegetation",
+    "occlusion",
+    "difficulty",
+    "context",
+)
+LINEAGE_SIDES = ("reference", "prediction")
+LINEAGE_FIELDS = ("source_id", "source_version", "derivation")
+NON_MEANINGFUL_TOKENS = {"", "unknown", "n/a", "na", "null", "tbd", "todo"}
+BELGIUM_SCOPE_BOUNDS = (1.9, 49.4, 7.5, 52.1)
+FAILURE_TAXONOMY = {
+    "M-FP-CONFUSER": "unmatched object or event prediction",
+    "M-FN-MISSED": "unmatched reference object or event",
+    "M-CLASS": "raster class differs from the valid reference class",
+    "M-BOUNDARY": "matched footprint has a measurable boundary deviation",
+    "M-AREA-BIAS": "matched footprint has a measurable relative area bias",
+    "M-MISCALIBRATED": "confidence diagnostic deviates from observed correctness",
+    "M-OOD": "error observed in an explicitly declared out-of-distribution case",
+    "D-VALIDATION-FP": "anomaly or severity reported without an exact reference match",
+    "D-VALIDATION-FN": "reference anomaly or severity not exactly reported",
+    "P-PARTIAL": "required prediction value is unavailable",
+}
+FAILURE_CONTEXTS = {
+    "tile_edge": "case metadata explicitly marks tile-edge context",
+    "high_confidence": (
+        "false positive meets the highest frozen diagnostic confidence threshold"
+    ),
+    "out_of_distribution": "case metadata explicitly marks OOD context",
 }
 
 
@@ -146,17 +196,62 @@ def _nonempty_mapping(value: Any, label: str) -> dict[str, Any]:
     return value
 
 
+def _meaningful_string(
+    value: Any,
+    label: str,
+    *,
+    allow_not_applicable: bool = False,
+) -> str:
+    if not isinstance(value, str) or value.strip().lower() in NON_MEANINGFUL_TOKENS:
+        raise ValueError(f"{label} must be a meaningful non-empty string")
+    normalized = value.strip()
+    if not allow_not_applicable and normalized.lower() == "not_applicable":
+        raise ValueError(f"{label} may not be not_applicable")
+    return normalized
+
+
+def _validate_lineage(case: dict[str, Any]) -> None:
+    sample_id = case["sample_id"]
+    lineage = _nonempty_mapping(case.get("lineage"), f"{sample_id}: lineage")
+    missing_sides = sorted(set(LINEAGE_SIDES) - lineage.keys())
+    if missing_sides:
+        raise ValueError(f"{sample_id}: lineage missing {missing_sides}")
+    for side in LINEAGE_SIDES:
+        entry = _nonempty_mapping(lineage.get(side), f"{sample_id}: lineage.{side}")
+        missing_fields = sorted(set(LINEAGE_FIELDS) - entry.keys())
+        if missing_fields:
+            raise ValueError(f"{sample_id}: lineage.{side} missing {missing_fields}")
+        for field in LINEAGE_FIELDS:
+            _meaningful_string(
+                entry.get(field),
+                f"{sample_id}: lineage.{side}.{field}",
+            )
+
+
 def _validate_case_common(case: dict[str, Any]) -> None:
     sample_id = case.get("sample_id")
     if not isinstance(sample_id, str) or not sample_id.strip():
         raise ValueError("Every case requires a non-empty sample_id")
     if case.get("task") not in TASKS:
         raise ValueError(f"{sample_id}: unsupported task {case.get('task')!r}")
-    if not isinstance(case.get("metadata"), dict):
+    metadata = _nonempty_mapping(case.get("metadata"), f"{sample_id}: metadata")
-        raise ValueError(f"{sample_id}: metadata must be an object")
+    missing_metadata = sorted(set(REQUIRED_METADATA_STRING_FIELDS) - metadata.keys())
+    if missing_metadata:
+        raise ValueError(f"{sample_id}: metadata missing {missing_metadata}")
+    for field in REQUIRED_METADATA_STRING_FIELDS:
+        _meaningful_string(
+            metadata.get(field),
+            f"{sample_id}: metadata.{field}",
+            allow_not_applicable=True,
+        )
+    resolution = metadata.get("resolution_m")
+    if case["task"] == "geospatial_data_validation" and resolution is None:
+        pass
+    elif _finite_float(resolution, f"{sample_id}: metadata.resolution_m") <= 0:
+        raise ValueError(f"{sample_id}: metadata.resolution_m must be positive")
     if not isinstance(case.get("config"), dict):
         raise ValueError(f"{sample_id}: config must be an explicit object")
-    _nonempty_mapping(case.get("lineage"), f"{sample_id}: lineage")
+    _validate_lineage(case)
 
 
 def _stable_id(item: dict[str, Any], label: str) -> str:
@@ -339,6 +434,8 @@ def detection_ap(
             reference = references[index]
             if not _class_compatible(prediction, reference):
                 continue
+            if prediction.get("_sample_id") != reference.get("_sample_id"):
+                continue
             candidates.append(
                 (
                     bbox_iou(prediction["bbox"], reference["bbox"]),
@@ -510,6 +607,18 @@ def evaluate_object_detection(case: dict[str, Any]) -> dict[str, Any]:
         case["config"].get("match_iou"),
         f"{case['sample_id']}.config.match_iou",
     )
+    diagnostic_thresholds = case["config"].get("fixed_diagnostic_risk_thresholds")
+    if diagnostic_thresholds is not None:
+        if not isinstance(diagnostic_thresholds, list) or not diagnostic_thresholds:
+            raise ValueError(
+                f"{case['sample_id']}: fixed diagnostic risk thresholds must be a "
+                "non-empty list"
+            )
+        for index, value in enumerate(diagnostic_thresholds):
+            _probability(
+                value,
+                f"{case['sample_id']}.config.fixed_diagnostic_risk_thresholds[{index}]",
+            )
     retained = [item for item in predictions if float(item["confidence"]) >= threshold]
     matches, false_positives, false_negatives = greedy_match(
         retained,
@@ -538,6 +647,10 @@ def evaluate_object_detection(case: dict[str, Any]) -> dict[str, Any]:
         class_metrics["ap50"] = detection_ap(
             all_class_predictions, class_references, 0.5
         )
+        class_metrics["ap50_95"] = _mean_or_none(
+            detection_ap(all_class_predictions, class_references, 0.5 + step * 0.05)
+            for step in range(10)
+        )
         per_class[str(class_value)] = class_metrics
     metrics = count_metrics(len(matches), len(false_positives), len(false_negatives))
     metrics.update(
@@ -555,9 +668,11 @@ def evaluate_object_detection(case: dict[str, Any]) -> dict[str, Any]:
             "coverage_risk": coverage_risk(
                 predictions, references, match_iou, threshold
             ),
+            "map50": _mean_or_none(item["ap50"] for item in per_class.values()),
+            "map50_95": _mean_or_none(item["ap50_95"] for item in per_class.values()),
         }
     )
-    return result(
+    evaluation = result(
         case,
         metrics,
         matches,
@@ -571,12 +686,24 @@ def evaluate_object_detection(case: dict[str, Any]) -> dict[str, Any]:
             "threshold": threshold,
         },
     )
-
+    calibration = metrics["calibration"]
-
+    if calibration["status"] == "computed" and calibration["ece"] > 1e-12:
-def polygon(item: dict[str, Any]):
+        evaluation["failures"].append(
-    from shapely.geometry import Polygon
+            failure_entry(
-
+                case,
-    return Polygon(item["polygon"])
+                "miscalibrated",
+                {
+                    "ece": calibration["ece"],
+                    "brier": calibration["brier"],
+                    "prediction_support": len(retained),
+                    "claim_boundary": (
+                        "diagnostic deviation only; not a representative "
+                        "population-calibration claim"
+                    ),
+                },
+            )
+        )
+    return evaluation
 
 
 def distribution(values: list[float]) -> dict[str, Any]:
@@ -594,6 +721,18 @@ def distribution(values: list[float]) -> dict[str, Any]:
     }
 
 
+def _bounds_overlap(
+    left: tuple[float, float, float, float],
+    right: tuple[float, float, float, float],
+) -> bool:
+    return not (
+        left[2] < right[0]
+        or left[0] > right[2]
+        or left[3] < right[1]
+        or left[1] > right[3]
+    )
+
+
 def _validate_metric_spatial_context(case: dict[str, Any]) -> dict[str, Any]:
     context = _nonempty_mapping(
         case.get("spatial_context"),
@@ -629,32 +768,99 @@ def _validate_metric_spatial_context(case: dict[str, Any]) -> dict[str, Any]:
         for axis in crs.axis_info[:2]
     ):
         raise ValueError(f"{case['sample_id']}: CRS axes must use metres")
+    if crs.to_epsg() in {3395, 3857}:
+        raise ValueError(
+            f"{case['sample_id']}: Web/World Mercator is unsuitable for "
+            "benchmark area, boundary and distance metrics"
+        )
+    area = crs.area_of_use
+    if area is None:
+        raise ValueError(
+            f"{case['sample_id']}: CRS area of use is unavailable; Belgian "
+            "metric suitability cannot be proven"
+        )
+    crs_bounds = (area.west, area.south, area.east, area.north)
+    if not _bounds_overlap(crs_bounds, BELGIUM_SCOPE_BOUNDS):
+        raise ValueError(
+            f"{case['sample_id']}: CRS area of use does not overlap the "
+            "declared Belgium and Belgian North Sea product scope"
+        )
     return context
 
 
-def _validated_polygon(item: dict[str, Any], label: str):
+def _validated_ring(coordinates: Any, label: str) -> list[list[float]]:
-    coordinates = item.get("polygon")
     if not isinstance(coordinates, list) or len(coordinates) < 4:
         raise ValueError(
-            f"{label}.polygon must contain a closed ring with at least four points"
+            f"{label} must contain a closed ring with at least four points"
         )
-    normalized: list[tuple[float, float]] = []
+    normalized: list[list[float]] = []
     for index, point in enumerate(coordinates):
         if not isinstance(point, list) or len(point) != 2:
-            raise ValueError(f"{label}.polygon[{index}] must be an [x, y] pair")
+            raise ValueError(f"{label}[{index}] must be an [x, y] pair")
         normalized.append(
-            (
+            [
-                _finite_float(point[0], f"{label}.polygon[{index}][0]"),
+                _finite_float(point[0], f"{label}[{index}][0]"),
-                _finite_float(point[1], f"{label}.polygon[{index}][1]"),
+                _finite_float(point[1], f"{label}[{index}][1]"),
-            )
+            ]
         )
     if normalized[0] != normalized[-1]:
-        raise ValueError(f"{label}.polygon ring must be explicitly closed")
+        raise ValueError(f"{label} must be explicitly closed")
-    from shapely.geometry import Polygon
+    return normalized
 
-    geometry = Polygon(normalized)
+
+def _validated_polygon(item: dict[str, Any], label: str):
+    if "geometry" in item and "polygon" in item:
+        raise ValueError(f"{label} may not declare both geometry and polygon")
+    if "geometry" in item:
+        payload = item["geometry"]
+        if not isinstance(payload, dict):
+            raise ValueError(f"{label}.geometry must be a GeoJSON object")
+        geometry_type = payload.get("type")
+        coordinates = payload.get("coordinates")
+        if geometry_type == "Polygon":
+            if not isinstance(coordinates, list) or not coordinates:
+                raise ValueError(f"{label}.geometry Polygon requires rings")
+            normalized_coordinates: Any = [
+                _validated_ring(ring, f"{label}.geometry.coordinates[{index}]")
+                for index, ring in enumerate(coordinates)
+            ]
+        elif geometry_type == "MultiPolygon":
+            if not isinstance(coordinates, list) or not coordinates:
+                raise ValueError(
+                    f"{label}.geometry MultiPolygon requires polygon members"
+                )
+            normalized_coordinates = []
+            for polygon_index, polygon_coordinates in enumerate(coordinates):
+                if not isinstance(polygon_coordinates, list) or not polygon_coordinates:
+                    raise ValueError(
+                        f"{label}.geometry.coordinates[{polygon_index}] requires rings"
+                    )
+                normalized_coordinates.append(
+                    [
+                        _validated_ring(
+                            ring,
+                            f"{label}.geometry.coordinates[{polygon_index}]"
+                            f"[{ring_index}]",
+                        )
+                        for ring_index, ring in enumerate(polygon_coordinates)
+                    ]
+                )
+        else:
+            raise ValueError(f"{label}.geometry.type must be Polygon or MultiPolygon")
+        normalized_payload = {
+            "type": geometry_type,
+            "coordinates": normalized_coordinates,
+        }
+    else:
+        normalized_payload = {
+            "type": "Polygon",
+            "coordinates": [_validated_ring(item.get("polygon"), f"{label}.polygon")],
+        }
+    from shapely.geometry import shape
+
+    geometry = shape(normalized_payload)
     if geometry.is_empty or geometry.area <= 0 or not geometry.is_valid:
-        raise ValueError(f"{label}.polygon must be non-empty, positive-area and valid")
+        raise ValueError(f"{label}.geometry must be non-empty, positive-area and valid")
     return geometry
 
 
@@ -679,16 +885,14 @@ def _validated_polygon_items(
     validated_references = [
         dict(
             item,
-            geometry=_validated_polygon(
+            _shape=_validated_polygon(item, f"{case['sample_id']}.references[{index}]"),
-                item, f"{case['sample_id']}.references[{index}]"
-            ),
         )
         for index, item in enumerate(references)
     ]
     validated_predictions = [
         dict(
             item,
-            geometry=_validated_polygon(
+            _shape=_validated_polygon(
                 item, f"{case['sample_id']}.predictions[{index}]"
             ),
         )
@@ -698,8 +902,8 @@ def _validated_polygon_items(
 
 
 def _polygon_overlap(prediction: dict[str, Any], reference: dict[str, Any]) -> float:
-    intersection = prediction["geometry"].intersection(reference["geometry"]).area
+    intersection = prediction["_shape"].intersection(reference["_shape"]).area
-    union = prediction["geometry"].union(reference["geometry"]).area
+    union = prediction["_shape"].union(reference["_shape"]).area
     return intersection / union if union > 0 else 0.0
 
 
@@ -753,13 +957,23 @@ def evaluate_footprint_segmentation(case: dict[str, Any]) -> dict[str, Any]:
     centroids = []
     area_errors = []
     for match in matches:
-        pred = by_prediction[match["prediction_id"]]["geometry"]
+        pred = by_prediction[match["prediction_id"]]["_shape"]
-        ref = by_reference[match["reference_id"]]["geometry"]
+        ref = by_reference[match["reference_id"]]["_shape"]
         intersection = pred.intersection(ref).area
-        dice.append(2 * intersection / (pred.area + ref.area))
+        dice_value = 2 * intersection / (pred.area + ref.area)
-        boundary.append(boundary_f1(pred, ref, tolerance))
+        boundary_value = boundary_f1(pred, ref, tolerance)
-        centroids.append(pred.centroid.distance(ref.centroid))
+        centroid_value = pred.centroid.distance(ref.centroid)
-        area_errors.append((pred.area - ref.area) / ref.area)
+        area_error = (pred.area - ref.area) / ref.area
+        dice.append(dice_value)
+        boundary.append(boundary_value)
+        centroids.append(centroid_value)
+        area_errors.append(area_error)
+        match.update(
+            dice=dice_value,
+            boundary_f1=boundary_value,
+            centroid_distance_m=centroid_value,
+            relative_area_error=area_error,
+        )
     metrics = count_metrics(len(matches), len(false_positives), len(false_negatives))
     metrics.update(
         {
@@ -773,13 +987,40 @@ def evaluate_footprint_segmentation(case: dict[str, Any]) -> dict[str, Any]:
             "spatial_context_validated": True,
         }
     )
-    return result(
+    evaluation = result(
         case,
         metrics,
         matches,
         false_positives,
         false_negatives,
     )
+    for match in matches:
+        if match["boundary_f1"] is not None and match["boundary_f1"] < 1 - 1e-12:
+            evaluation["failures"].append(
+                failure_entry(
+                    case,
+                    "boundary_error",
+                    {
+                        "prediction_id": match["prediction_id"],
+                        "reference_id": match["reference_id"],
+                        "boundary_f1": match["boundary_f1"],
+                        "boundary_tolerance_m": tolerance,
+                    },
+                )
+            )
+        if abs(match["relative_area_error"]) > 1e-12:
+            evaluation["failures"].append(
+                failure_entry(
+                    case,
+                    "area_bias",
+                    {
+                        "prediction_id": match["prediction_id"],
+                        "reference_id": match["reference_id"],
+                        "relative_area_error": match["relative_area_error"],
+                    },
+                )
+            )
+    return evaluation
 
 
 def _validate_rectangular_grid(
@@ -836,6 +1077,14 @@ def _validate_raster_side(
         )
         for index, value in enumerate(transform)
     )
+    a, b, _x_offset, d, e, _y_offset = normalized_transform
+    determinant = a * e - b * d
+    linear_scale = max(abs(a), abs(b), abs(d), abs(e), 1.0)
+    if abs(determinant) <= 1e-12 * linear_scale * linear_scale:
+        raise ValueError(
+            f"{case['sample_id']}: raster affine transform is singular or "
+            "numerically invalid"
+        )
     shape = context["shape"]
     if (
         not isinstance(shape, list)
@@ -1195,9 +1444,22 @@ def evaluate_validation(case: dict[str, Any]) -> dict[str, Any]:
     observed_items = _normalized_anomalies(case, "observed_anomalies")
     expected = {item["code"]: item for item in expected_items}
     observed = {item["code"]: item for item in observed_items}
-    matched_codes = sorted(expected.keys() & observed.keys())
+    common_codes = sorted(expected.keys() & observed.keys())
-    false_positive_codes = sorted(observed.keys() - expected.keys())
+    matched_codes = [
-    false_negative_codes = sorted(expected.keys() - observed.keys())
+        code
+        for code in common_codes
+        if expected[code]["severity"] == observed[code]["severity"]
+    ]
+    severity_mismatches = [
+        {
+            "anomaly": code,
+            "expected_severity": expected[code]["severity"],
+            "observed_severity": observed[code]["severity"],
+            "reason": "severity_mismatch",
+        }
+        for code in common_codes
+        if expected[code]["severity"] != observed[code]["severity"]
+    ]
     matched = [
         {
             "anomaly": code,
@@ -1208,13 +1470,31 @@ def evaluate_validation(case: dict[str, Any]) -> dict[str, Any]:
     ]
     false_positives = [
         {"anomaly": code, "severity": observed[code]["severity"]}
-        for code in false_positive_codes
+        for code in sorted(observed.keys() - expected.keys())
+    ] + [
+        {
+            "anomaly": item["anomaly"],
+            "severity": item["observed_severity"],
+            "expected_severity": item["expected_severity"],
+            "reason": "severity_mismatch",
+        }
+        for item in severity_mismatches
     ]
     false_negatives = [
         {"anomaly": code, "severity": expected[code]["severity"]}
-        for code in false_negative_codes
+        for code in sorted(expected.keys() - observed.keys())
+    ] + [
+        {
+            "anomaly": item["anomaly"],
+            "severity": item["expected_severity"],
+            "observed_severity": item["observed_severity"],
+            "reason": "severity_mismatch",
+        }
+        for item in severity_mismatches
     ]
     metrics = count_metrics(len(matched), len(false_positives), len(false_negatives))
+    metrics["severity_mismatch_count"] = len(severity_mismatches)
+    metrics["severity_mismatches"] = severity_mismatches
     metrics["blocker_or_critical_miss_count"] = sum(
         item["severity"] in {"blocker", "critical"} for item in false_negatives
     )
@@ -1259,6 +1539,7 @@ def result(
         "input_lineage": serializable(case["lineage"]),
         "reference_input_field": reference_field,
         "prediction_input_field": prediction_field,
+        "classes": serializable(case.get("classes")),
         "references": exact_references,
         "predictions_pre_filter": exact_predictions,
         "predictions_post_filter": post_filter,
@@ -1297,7 +1578,7 @@ def result(
 def serializable(value: Any) -> Any:
     if isinstance(value, dict):
         return {
-            key: serializable(item) for key, item in value.items() if key != "geometry"
+            key: serializable(item) for key, item in value.items() if key != "_shape"
         }
     if isinstance(value, list):
         return [serializable(item) for item in value]
@@ -1309,34 +1590,79 @@ def serializable(value: Any) -> Any:
 def failure_entry(
     case: dict[str, Any], kind: str, evidence: dict[str, Any]
 ) -> dict[str, Any]:
+    base_kind = kind
+    if base_kind in {"false_positive", "false_negative"}:
         if case["task"] == "change_detection":
             kind = (
                 "event_false_positive"
-            if kind == "false_positive"
+                if base_kind == "false_positive"
                 else "event_false_negative"
             )
         elif case["task"] == "geospatial_data_validation":
             kind = (
                 "validation_false_positive"
-            if kind == "false_positive"
+                if base_kind == "false_positive"
                 else "validation_false_negative"
             )
         elif case["task"] == "terrain_interpretation":
             kind = "terrain_missing"
+        elif case["task"] == "raster_classification":
+            kind = (
+                "raster_misclassification"
+                if evidence.get("reason") == "class_mismatch"
+                else "raster_missing"
+            )
+
+    contexts: list[str] = []
+    secondary_error_codes: list[str] = []
+    metadata = case.get("metadata", {})
+    if metadata.get("tile_edge") is True:
+        contexts.append("tile_edge")
+    if metadata.get("ood") is True:
+        contexts.append("out_of_distribution")
+        secondary_error_codes.append("M-OOD")
+    confidence = evidence.get("confidence")
+    diagnostic_thresholds = case.get("config", {}).get(
+        "fixed_diagnostic_risk_thresholds"
+    )
+    valid_thresholds = (
+        [
+            float(value)
+            for value in diagnostic_thresholds
+            if isinstance(value, (int, float))
+            and not isinstance(value, bool)
+            and math.isfinite(float(value))
+            and 0 <= float(value) <= 1
+        ]
+        if isinstance(diagnostic_thresholds, list)
+        else []
+    )
+    if (
+        base_kind == "false_positive"
+        and isinstance(confidence, (int, float))
+        and not isinstance(confidence, bool)
+        and valid_thresholds
+        and float(confidence) >= max(valid_thresholds)
+    ):
+        contexts.append("high_confidence")
+        secondary_error_codes.append("M-MISCALIBRATED")
+    normalized_evidence = serializable(evidence)
     return {
         "failure_id": canonical_hash(
             {
                 "sample_id": case["sample_id"],
                 "kind": kind,
-                "evidence": serializable(evidence),
+                "evidence": normalized_evidence,
             }
         )[:20],
         "sample_id": case["sample_id"],
         "task": case["task"],
         "error_code": ERROR_CODES[kind],
+        "secondary_error_codes": sorted(set(secondary_error_codes)),
+        "contexts": sorted(set(contexts)),
         "kind": kind,
-        "metadata": case["metadata"],
+        "metadata": serializable(metadata),
-        "evidence": serializable(evidence),
+        "evidence": normalized_evidence,
     }
 
 
@@ -1627,6 +1953,127 @@ def _aggregate_count_family(
     }
 
 
+def _pooled_detection_inputs(
+    values: list[dict[str, Any]],
+) -> tuple[
+    list[dict[str, Any]],
+    list[dict[str, Any]],
+    list[dict[str, Any]],
+    list[dict[str, Any]],
+    list[str | int],
+]:
+    references: list[dict[str, Any]] = []
+    predictions_pre_filter: list[dict[str, Any]] = []
+    predictions_post_filter: list[dict[str, Any]] = []
+    matches: list[dict[str, Any]] = []
+    classes_by_hash: dict[str, str | int] = {}
+    for value in sorted(values, key=lambda item: item.get("sample_id", "")):
+        sample_id = value.get("sample_id")
+        if not isinstance(sample_id, str) or not sample_id:
+            raise ValueError("Pooled detection metrics require a sample_id")
+        raw = _nonempty_mapping(
+            value.get("raw"), f"{sample_id}: pooled detection raw evidence"
+        )
+        required_lists = (
+            "references",
+            "predictions_pre_filter",
+            "predictions_post_filter",
+            "matches",
+            "classes",
+        )
+        for field in required_lists:
+            if not isinstance(raw.get(field), list):
+                raise ValueError(
+                    f"{sample_id}: pooled detection raw.{field} must be a list"
+                )
+        for class_value in raw["classes"]:
+            classes_by_hash[canonical_hash(class_value)] = class_value
+
+        def scoped(item: dict[str, Any], label: str) -> dict[str, Any]:
+            if not isinstance(item, dict):
+                raise ValueError(f"{sample_id}: pooled {label} must contain objects")
+            identifier = _stable_id(item, f"{sample_id}: pooled {label}")
+            return {
+                **item,
+                "id": f"{sample_id}::{identifier}",
+                "_sample_id": sample_id,
+            }
+
+        references.extend(scoped(item, "references") for item in raw["references"])
+        predictions_pre_filter.extend(
+            scoped(item, "predictions_pre_filter")
+            for item in raw["predictions_pre_filter"]
+        )
+        predictions_post_filter.extend(
+            scoped(item, "predictions_post_filter")
+            for item in raw["predictions_post_filter"]
+        )
+        for match in raw["matches"]:
+            if not isinstance(match, dict):
+                raise ValueError(f"{sample_id}: pooled matches must contain objects")
+            prediction_id = match.get("prediction_id")
+            if not isinstance(prediction_id, str) or not prediction_id:
+                raise ValueError(
+                    f"{sample_id}: pooled match prediction_id must be non-empty"
+                )
+            matches.append({**match, "prediction_id": f"{sample_id}::{prediction_id}"})
+    return (
+        references,
+        predictions_pre_filter,
+        predictions_post_filter,
+        matches,
+        [classes_by_hash[key] for key in sorted(classes_by_hash)],
+    )
+
+
+def _aggregate_detection(values: list[dict[str, Any]]) -> dict[str, Any]:
+    aggregation = _aggregate_count_family(values)
+    references, predictions, retained, matches, classes = _pooled_detection_inputs(
+        values
+    )
+    per_class = {}
+    for class_value in classes:
+        class_references = [item for item in references if item["class"] == class_value]
+        class_predictions = [
+            item for item in predictions if item["class"] == class_value
+        ]
+        ap50 = detection_ap(class_predictions, class_references, 0.5)
+        ap50_95 = _mean_or_none(
+            detection_ap(class_predictions, class_references, 0.5 + step * 0.05)
+            for step in range(10)
+        )
+        per_class[str(class_value)] = {
+            "reference_count": len(class_references),
+            "prediction_count": len(class_predictions),
+            "ap50": ap50,
+            "ap50_95": ap50_95,
+        }
+    aggregation["micro"].update(
+        {
+            "ap50": detection_ap(predictions, references, 0.5),
+            "ap50_95": _mean_or_none(
+                detection_ap(predictions, references, 0.5 + step * 0.05)
+                for step in range(10)
+            ),
+            "map50": _mean_or_none(item["ap50"] for item in per_class.values()),
+            "map50_95": _mean_or_none(item["ap50_95"] for item in per_class.values()),
+            "per_class_ap": per_class,
+            "calibration": calibration_metrics(retained, matches),
+            "ranking_scope": (
+                "predictions pooled across cases; class- and sample-aware matching; "
+                "no averaging of per-case AP"
+            ),
+        }
+    )
+    aggregation["observation_support"].update(
+        {
+            "ranking_predictions": len(predictions),
+            "calibration_predictions": len(retained),
+        }
+    )
+    return aggregation
+
+
 def _aggregate_raster(values: list[dict[str, Any]]) -> dict[str, Any]:
     metrics = [item["metrics"] for item in values]
     class_counts: dict[str, dict[str, int]] = defaultdict(
@@ -1727,13 +2174,14 @@ def _aggregate_terrain(values: list[dict[str, Any]]) -> dict[str, Any]:
 
 def _aggregate_task(values: list[dict[str, Any]]) -> dict[str, Any]:
     task = values[0]["task"]
-    if task == "raster_classification":
+    if task == "object_detection":
+        aggregation = _aggregate_detection(values)
+    elif task == "raster_classification":
         aggregation = _aggregate_raster(values)
     elif task == "terrain_interpretation":
         aggregation = _aggregate_terrain(values)
     else:
         extras = {
-            "object_detection": ("ap50", "ap50_95"),
             "footprint_segmentation": (
                 "mean_iou",
                 "mean_dice",
@@ -1827,6 +2275,7 @@ def subgroup_report(results: list[dict[str, Any]]) -> dict[str, Any]:
         "vegetation",
         "occlusion",
         "difficulty",
+        "context",
     )
     dimension_reports = {}
     any_insufficient = False
@@ -1881,12 +2330,32 @@ def evaluate_cases(path: Path, allowed_sample_ids: set[str]) -> dict[str, Any]:
     portfolio = json.loads(path.read_text(encoding="utf-8"))
     if not isinstance(portfolio, dict):
         raise ValueError("The evaluation portfolio must be a JSON object")
-    claim_boundary = portfolio.get("claim_boundary")
+    schema_version = portfolio.get("schema_version")
-    if not isinstance(claim_boundary, str) or (
+    if type(schema_version) is not int or schema_version != REPORT_SCHEMA_VERSION:
-        "synthetic" not in claim_boundary.lower()
-    ):
         raise ValueError(
-            "The portfolio claim boundary must explicitly state that it is synthetic"
+            f"schema_version must be exactly integer {REPORT_SCHEMA_VERSION}"
+        )
+    _meaningful_string(portfolio.get("portfolio_id"), "portfolio_id")
+    protected_policy = portfolio.get("protected_policy")
+    if protected_policy != EXPECTED_PROTECTED_POLICY:
+        raise ValueError(
+            "protected_policy must exactly equal the frozen no-selection policy: "
+            f"{EXPECTED_PROTECTED_POLICY}"
+        )
+    portfolio_kind = portfolio.get("portfolio_kind")
+    if portfolio_kind not in PORTFOLIO_KINDS:
+        raise ValueError(f"portfolio_kind must be one of {sorted(PORTFOLIO_KINDS)}")
+    claim_boundary = portfolio.get("claim_boundary")
+    claim_lower = claim_boundary.lower() if isinstance(claim_boundary, str) else ""
+    if portfolio_kind == "synthetic_contract":
+        if "synthetic" not in claim_lower:
+            raise ValueError(
+                "A synthetic_contract claim boundary must explicitly state that it is synthetic"
+            )
+    elif "governed product baseline" not in claim_lower or "synthetic" in claim_lower:
+        raise ValueError(
+            "A governed_product_baseline claim boundary must explicitly state "
+            "'governed product baseline' and must not describe the portfolio as synthetic"
         )
     split_roles = portfolio.get("split_roles")
     if split_roles not in (["test"], ["test", "background-test"]):
@@ -1895,17 +2364,18 @@ def evaluate_cases(path: Path, allowed_sample_ids: set[str]) -> dict[str, Any]:
         )
     if "challenge_cases" in portfolio or "challenge_labels" in portfolio:
         raise ValueError("Challenge cases and labels must remain sealed and absent")
-    selection_policy = portfolio.get("selection_policy")
+    selection_policy = _meaningful_string(
-    if not isinstance(selection_policy, str) or not selection_policy.strip():
+        portfolio.get("selection_policy"), "selection_policy"
-        raise ValueError("The portfolio requires an explicit selection policy")
+    )
     declared_portfolio_lineage = _nonempty_mapping(
         portfolio.get("portfolio_lineage"),
         "portfolio_lineage",
     )
     for field in ("origin", "source_path", "version"):
-        value = declared_portfolio_lineage.get(field)
+        _meaningful_string(
-        if not isinstance(value, str) or not value.strip():
+            declared_portfolio_lineage.get(field),
-            raise ValueError(f"portfolio_lineage.{field} must be a non-empty string")
+            f"portfolio_lineage.{field}",
+        )
     cases = portfolio.get("cases")
     if not isinstance(cases, list) or not all(isinstance(item, dict) for item in cases):
         raise ValueError("The portfolio cases must be an object list")
@@ -1923,6 +2393,13 @@ def evaluate_cases(path: Path, allowed_sample_ids: set[str]) -> dict[str, Any]:
     if duplicates:
         raise ValueError(f"Duplicate protected case ids: {duplicates}")
     case_ids = set(identifiers)
+    observed_tasks = {item["task"] for item in cases}
+    if portfolio_kind == "governed_product_baseline" and observed_tasks != TASKS:
+        raise ValueError(
+            "A governed_product_baseline must cover every evaluator task family: "
+            f"missing={sorted(TASKS - observed_tasks)}, "
+            f"unexpected={sorted(observed_tasks - TASKS)}"
+        )
     unexpected = sorted(case_ids - allowed_sample_ids)
     missing = sorted(allowed_sample_ids - case_ids)
     if unexpected or missing:
@@ -1939,6 +2416,7 @@ def evaluate_cases(path: Path, allowed_sample_ids: set[str]) -> dict[str, Any]:
         "declared": serializable(declared_portfolio_lineage),
         "portfolio_id": portfolio.get("portfolio_id"),
         "portfolio_schema_version": portfolio.get("schema_version"),
+        "portfolio_kind": portfolio_kind,
         "portfolio_file_sha256": file_hash,
         "portfolio_canonical_json_sha256": portfolio_canonical_hash,
         "source_path": declared_portfolio_lineage["source_path"],
@@ -1947,6 +2425,13 @@ def evaluate_cases(path: Path, allowed_sample_ids: set[str]) -> dict[str, Any]:
     }
     for item in results:
         item["raw"]["portfolio_lineage"] = portfolio_lineage
+    results_by_task: dict[str, list[dict[str, Any]]] = defaultdict(list)
+    for item in results:
+        results_by_task[item["task"]].append(item)
+    portfolio_metrics = {
+        task: _aggregate_task(task_results)
+        for task, task_results in sorted(results_by_task.items())
+    }
     failures = sorted(
         (failure for item in results for failure in item["failures"]),
         key=lambda item: item["failure_id"],
@@ -1956,6 +2441,7 @@ def evaluate_cases(path: Path, allowed_sample_ids: set[str]) -> dict[str, Any]:
         "schema_version": REPORT_SCHEMA_VERSION,
         "evaluator_version": EVALUATOR_VERSION,
         "portfolio_id": portfolio["portfolio_id"],
+        "portfolio_kind": portfolio_kind,
         "portfolio_file_sha256": file_hash,
         "portfolio_canonical_json_sha256": portfolio_canonical_hash,
         "claim_boundary": claim_boundary,
@@ -1978,5 +2464,11 @@ def evaluate_cases(path: Path, allowed_sample_ids: set[str]) -> dict[str, Any]:
         "results": results,
         "subgroups": subgroup_report(results),
         "failures": failures,
+        "failure_taxonomy": {
+            "primary_codes": FAILURE_TAXONOMY,
+            "contexts": FAILURE_CONTEXTS,
+            "claim_boundary": "diagnostic classification; not a release decision",
+        },
+        "portfolio_metrics": portfolio_metrics,
         "results_canonical_json_sha256": results_hash,
     }