feat(accuracy): build hardened phase 4 evaluation harness

2026-08-02 02:43:41 +02:00
parent 7b92e29e49
commit bd2fd9780f
13 changed files with 5468 additions and 7 deletions
@@ -0,0 +1,222 @@
+from __future__ import annotations
+
+import copy
+import json
+import sys
+from pathlib import Path
+
+import pytest
+
+
+ROOT = Path(__file__).resolve().parents[2]
+SCRIPTS = ROOT / "scripts"
+if str(SCRIPTS) not in sys.path:
+    sys.path.insert(0, str(SCRIPTS))
+
+from generate_accuracy_phase4_splits import (  # noqa: E402
+    LeakageError,
+    assert_training_inputs_safe,
+    build_manifests,
+    generate,
+)
+
+
+SOURCE = ROOT / "fixtures/accuracy/p4/split-source-manifest.json"
+
+
+def load_source() -> dict:
+    return json.loads(SOURCE.read_text(encoding="utf-8"))
+
+
+def test_normative_roles_hashes_and_source_order_are_enforced() -> None:
+    source = load_source()
+    development, protected, leakage = build_manifests(source)
+    reversed_source = copy.deepcopy(source)
+    reversed_source["samples"].reverse()
+    reversed_development, reversed_protected, reversed_leakage = build_manifests(
+        reversed_source
+    )
+
+    assert leakage["status"] == "pass"
+    assert leakage["finding_count"] == 0
+    assert leakage["split_counts"] == {
+        "background-test": 2,
+        "calibration": 2,
+        "challenge": 4,
+        "test": 7,
+        "train": 3,
+        "val": 3,
+    }
+    assert leakage["crs_validation"] == {
+        "status": "pass",
+        "crs": "EPSG:31370",
+        "distance_units": "m",
+    }
+    assert development["training_access_allowed_by_split"] == {
+        "train": True,
+        "val": False,
+        "calibration": False,
+    }
+    assert protected["labels_available_by_split"]["challenge"] == "sealed_external"
+    assert reversed_development["manifest_sha256"] == development["manifest_sha256"]
+    assert reversed_protected["manifest_sha256"] == protected["manifest_sha256"]
+    assert reversed_leakage == leakage
+
+
+@pytest.mark.parametrize(
+    ("field", "expected_code"),
+    [
+        ("group_id", "S-SPATIAL-GROUP"),
+        ("source_family", "S-SOURCE-FAMILY"),
+        ("temporal_family", "S-TEMPORAL-FAMILY"),
+        ("raw_image_sha256", "S-RAW-IMAGE-DUPLICATE"),
+        ("processed_image_sha256", "S-PROCESSED-IMAGE-DUPLICATE"),
+        ("label_sha256", "S-LABEL-DUPLICATE"),
+        ("label_geometry_hash", "S-LABEL-GEOMETRY-DUPLICATE"),
+        ("parent_raster_id", "S-PARENT-RASTER"),
+        ("acquisition_id", "S-ACQUISITION"),
+    ],
+)
+def test_cross_split_lineage_and_content_collisions_fail(
+    field: str, expected_code: str
+) -> None:
+    source = load_source()
+    source["samples"][8][field] = source["samples"][0][field]
+    _development, _protected, leakage = build_manifests(source)
+
+    assert leakage["status"] == "fail"
+    assert expected_code in {item["code"] for item in leakage["findings"]}
+
+
+@pytest.mark.parametrize(
+    ("field", "expected_code"),
+    [
+        ("perceptual_image_hash", "S-PERCEPTUAL-IMAGE-NEAR-DUPLICATE"),
+        ("label_geometry_fingerprint", "S-LABEL-GEOMETRY-NEAR-DUPLICATE"),
+    ],
+)
+def test_near_duplicate_fingerprints_fail(field: str, expected_code: str) -> None:
+    source = load_source()
+    source["samples"][8][field] = source["samples"][0][field]
+    _development, _protected, leakage = build_manifests(source)
+
+    assert leakage["status"] == "fail"
+    assert expected_code in {item["code"] for item in leakage["findings"]}
+
+
+def test_object_native_feature_and_spatial_collisions_fail() -> None:
+    source = load_source()
+    source["samples"][8]["object_ids"] = source["samples"][0]["object_ids"]
+    source["samples"][9]["native_feature_ids"] = source["samples"][1][
+        "native_feature_ids"
+    ]
+    source["samples"][10]["bbox"] = source["samples"][2]["bbox"]
+    _development, _protected, leakage = build_manifests(source)
+    codes = {item["code"] for item in leakage["findings"]}
+
+    assert {"S-OBJECT-INSTANCE", "S-NATIVE-FEATURE", "S-SPATIAL-OVERLAP"} <= codes
+
+
+def test_non_metric_crs_and_missing_normative_role_fail_closed() -> None:
+    geographic = load_source()
+    geographic["crs"] = "EPSG:4326"
+    with pytest.raises(LeakageError, match="projected in metres"):
+        build_manifests(geographic)
+
+    missing = load_source()
+    missing["samples"] = [
+        item for item in missing["samples"] if item["split"] != "calibration"
+    ]
+    with pytest.raises(LeakageError, match="Required splits are absent"):
+        build_manifests(missing)
+
+
+def test_training_firewall_only_allows_train_and_binds_protected_lineage() -> None:
+    development, protected, leakage = build_manifests(load_source())
+    assert leakage["status"] == "pass"
+    train = [item for item in development["samples"] if item["split"] == "train"]
+    validation = next(item for item in development["samples"] if item["split"] == "val")
+    protected_item = protected["samples"][0]
+
+    assert_training_inputs_safe([], train, protected)
+    with pytest.raises(LeakageError, match="non_train_role"):
+        assert_training_inputs_safe([], [validation], protected)
+    with pytest.raises(LeakageError, match="protected_identity"):
+        disguised = copy.deepcopy(train[0])
+        disguised["source_family"] = protected_item["source_family"]
+        assert_training_inputs_safe([], [disguised], protected)
+    with pytest.raises(LeakageError, match="protected_path"):
+        assert_training_inputs_safe([Path("vault/protected/test.json")], [], protected)
+
+
+def test_failed_generation_writes_status_but_no_consumable_manifests(
+    tmp_path: Path,
+) -> None:
+    source = load_source()
+    source["samples"][8]["group_id"] = source["samples"][0]["group_id"]
+    source_path = tmp_path / "source.json"
+    source_path.write_text(json.dumps(source), encoding="utf-8")
+    output = tmp_path / "out"
+
+    with pytest.raises(LeakageError, match="Leakage gate failed"):
+        generate(source_path, output)
+
+    status = json.loads((output / "generation-status.json").read_text(encoding="utf-8"))
+    assert status["status"] == "fail"
+    assert not (output / "development-split-manifest.json").exists()
+    assert not (output / "protected-split-manifest.json").exists()
+
+
+def test_deterministic_grouped_assignment_is_stable_and_keeps_relatives_together() -> (
+    None
+):
+    source = load_source()
+    source["assignment_mode"] = "deterministic_grouped"
+    source["split_assignment"] = {
+        "seed": "fixed-phase4-test-seed",
+        "roles": [
+            "train",
+            "val",
+            "calibration",
+            "test",
+            "background-test",
+            "challenge",
+        ],
+        "weights": {
+            "train": 6,
+            "val": 2,
+            "calibration": 1,
+            "test": 2,
+            "background-test": 1,
+            "challenge": 1,
+        },
+        "stratify_by": ["task"],
+    }
+    for item in source["samples"]:
+        item.pop("split")
+    source["samples"][1]["group_id"] = source["samples"][0]["group_id"]
+
+    development, protected, leakage = build_manifests(source)
+    reversed_source = copy.deepcopy(source)
+    reversed_source["samples"].reverse()
+    reversed_development, reversed_protected, reversed_leakage = build_manifests(
+        reversed_source
+    )
+
+    assigned = {
+        item["sample_id"]: item["split"]
+        for item in development["samples"] + protected["samples"]
+    }
+    assert assigned["det-train-a"] == assigned["seg-train-a"]
+    assert set(leakage["split_counts"]) == {
+        "train",
+        "val",
+        "calibration",
+        "test",
+        "background-test",
+        "challenge",
+    }
+    assert leakage["status"] == "pass"
+    assert reversed_development["manifest_sha256"] == development["manifest_sha256"]
+    assert reversed_protected["manifest_sha256"] == protected["manifest_sha256"]
+    assert reversed_leakage == leakage