from __future__ import annotations import copy import json import sys from pathlib import Path import pytest ROOT = Path(__file__).resolve().parents[2] SCRIPTS = ROOT / "scripts" if str(SCRIPTS) not in sys.path: sys.path.insert(0, str(SCRIPTS)) from generate_accuracy_phase4_splits import ( # noqa: E402 LeakageError, assert_training_inputs_safe, build_manifests, generate, ) SOURCE = ROOT / "fixtures/accuracy/p4/split-source-manifest.json" def load_source() -> dict: return json.loads(SOURCE.read_text(encoding="utf-8")) def test_normative_roles_hashes_and_source_order_are_enforced() -> None: source = load_source() development, protected, leakage = build_manifests(source) reversed_source = copy.deepcopy(source) reversed_source["samples"].reverse() reversed_development, reversed_protected, reversed_leakage = build_manifests( reversed_source ) assert leakage["status"] == "pass" assert leakage["finding_count"] == 0 assert leakage["split_counts"] == { "background-test": 2, "calibration": 2, "challenge": 4, "test": 7, "train": 3, "val": 3, } assert leakage["crs_validation"] == { "status": "pass", "crs": "EPSG:31370", "distance_units": "m", } assert development["training_access_allowed_by_split"] == { "train": True, "val": False, "calibration": False, } assert protected["labels_available_by_split"]["challenge"] == "sealed_external" assert reversed_development["manifest_sha256"] == development["manifest_sha256"] assert reversed_protected["manifest_sha256"] == protected["manifest_sha256"] assert reversed_leakage == leakage @pytest.mark.parametrize( ("field", "expected_code"), [ ("group_id", "S-SPATIAL-GROUP"), ("source_family", "S-SOURCE-FAMILY"), ("temporal_family", "S-TEMPORAL-FAMILY"), ("raw_image_sha256", "S-RAW-IMAGE-DUPLICATE"), ("processed_image_sha256", "S-PROCESSED-IMAGE-DUPLICATE"), ("label_sha256", "S-LABEL-DUPLICATE"), ("label_geometry_hash", "S-LABEL-GEOMETRY-DUPLICATE"), ("parent_raster_id", "S-PARENT-RASTER"), ("acquisition_id", "S-ACQUISITION"), ], ) def test_cross_split_lineage_and_content_collisions_fail( field: str, expected_code: str ) -> None: source = load_source() source["samples"][8][field] = source["samples"][0][field] _development, _protected, leakage = build_manifests(source) assert leakage["status"] == "fail" assert expected_code in {item["code"] for item in leakage["findings"]} @pytest.mark.parametrize( ("field", "expected_code"), [ ("perceptual_image_hash", "S-PERCEPTUAL-IMAGE-NEAR-DUPLICATE"), ("label_geometry_fingerprint", "S-LABEL-GEOMETRY-NEAR-DUPLICATE"), ], ) def test_near_duplicate_fingerprints_fail(field: str, expected_code: str) -> None: source = load_source() source["samples"][8][field] = source["samples"][0][field] _development, _protected, leakage = build_manifests(source) assert leakage["status"] == "fail" assert expected_code in {item["code"] for item in leakage["findings"]} def test_object_native_feature_and_spatial_collisions_fail() -> None: source = load_source() source["samples"][8]["object_ids"] = source["samples"][0]["object_ids"] source["samples"][9]["native_feature_ids"] = source["samples"][1][ "native_feature_ids" ] source["samples"][10]["bbox"] = source["samples"][2]["bbox"] _development, _protected, leakage = build_manifests(source) codes = {item["code"] for item in leakage["findings"]} assert {"S-OBJECT-INSTANCE", "S-NATIVE-FEATURE", "S-SPATIAL-OVERLAP"} <= codes def test_non_metric_crs_and_missing_normative_role_fail_closed() -> None: geographic = load_source() geographic["crs"] = "EPSG:4326" with pytest.raises(LeakageError, match="projected in metres"): build_manifests(geographic) missing = load_source() missing["samples"] = [ item for item in missing["samples"] if item["split"] != "calibration" ] with pytest.raises(LeakageError, match="Required splits are absent"): build_manifests(missing) def test_training_firewall_only_allows_train_and_binds_protected_lineage() -> None: development, protected, leakage = build_manifests(load_source()) assert leakage["status"] == "pass" train = [item for item in development["samples"] if item["split"] == "train"] validation = next(item for item in development["samples"] if item["split"] == "val") protected_item = protected["samples"][0] assert_training_inputs_safe([], train, protected) with pytest.raises(LeakageError, match="non_train_role"): assert_training_inputs_safe([], [validation], protected) with pytest.raises(LeakageError, match="protected_identity"): disguised = copy.deepcopy(train[0]) disguised["source_family"] = protected_item["source_family"] assert_training_inputs_safe([], [disguised], protected) with pytest.raises(LeakageError, match="protected_path"): assert_training_inputs_safe([Path("vault/protected/test.json")], [], protected) def test_failed_generation_writes_status_but_no_consumable_manifests( tmp_path: Path, ) -> None: source = load_source() source["samples"][8]["group_id"] = source["samples"][0]["group_id"] source_path = tmp_path / "source.json" source_path.write_text(json.dumps(source), encoding="utf-8") output = tmp_path / "out" with pytest.raises(LeakageError, match="Leakage gate failed"): generate(source_path, output) status = json.loads((output / "generation-status.json").read_text(encoding="utf-8")) assert status["status"] == "fail" assert not (output / "development-split-manifest.json").exists() assert not (output / "protected-split-manifest.json").exists() def test_deterministic_grouped_assignment_is_stable_and_keeps_relatives_together() -> ( None ): source = load_source() source["assignment_mode"] = "deterministic_grouped" source["split_assignment"] = { "seed": "fixed-phase4-test-seed", "roles": [ "train", "val", "calibration", "test", "background-test", "challenge", ], "weights": { "train": 6, "val": 2, "calibration": 1, "test": 2, "background-test": 1, "challenge": 1, }, "stratify_by": ["task"], } for item in source["samples"]: item.pop("split") source["samples"][1]["group_id"] = source["samples"][0]["group_id"] development, protected, leakage = build_manifests(source) reversed_source = copy.deepcopy(source) reversed_source["samples"].reverse() reversed_development, reversed_protected, reversed_leakage = build_manifests( reversed_source ) assigned = { item["sample_id"]: item["split"] for item in development["samples"] + protected["samples"] } assert assigned["det-train-a"] == assigned["seg-train-a"] assert set(leakage["split_counts"]) == { "train", "val", "calibration", "test", "background-test", "challenge", } assert leakage["status"] == "pass" assert reversed_development["manifest_sha256"] == development["manifest_sha256"] assert reversed_protected["manifest_sha256"] == protected["manifest_sha256"] assert reversed_leakage == leakage