Jens/geointel
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(accuracy): close phase 4 evidence bypasses

Browse Source
This commit is contained in:
Jens
2026-08-02 04:29:10 +02:00
parent bd2fd9780f
commit ee1982eff2
7 changed files with 5814 additions and 382 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/tests/test_accuracy_phase4_evaluation.py
+981 -12
View File
File diff suppressed because it is too large Load Diff
backend/tests/test_accuracy_phase4_evaluator_hardening.py
+326 -5
View File
@@ -17,6 +17,7 @@ if str(SCRIPTS) not in sys.path:
from accuracy_phase4_evaluator import ( # noqa: E402
TASKS,
EXPECTED_PROTECTED_POLICY,
canonical_hash,
count_metrics,
detection_ap,
@@ -40,6 +41,7 @@ METADATA = {
"source": "synthetic-source",
"sensor": "synthetic-sensor",
"resolution_m": 0.25,
"context": "dense_urban",
"season": "summer",
"date": "2026-01-01",
"vegetation": "partial",
@@ -149,6 +151,7 @@ def test_raw_evidence_and_hashes_are_exact_and_recomputable(tmp_path: Path) -> N
case = detection_case()
portfolio = {
"schema_version": 2,
"portfolio_kind": "synthetic_contract",
"portfolio_id": "synthetic-hardening-test",
"portfolio_lineage": {
"origin": "repository_fixture",
@@ -158,7 +161,7 @@ def test_raw_evidence_and_hashes_are_exact_and_recomputable(tmp_path: Path) -> N
"split_roles": ["test"],
"selection_policy": "Fixed before evaluation; no selection.",
"claim_boundary": "Synthetic evaluator test; not product accuracy.",
"protected_policy": {"threshold_selection_allowed": False},
"protected_policy": copy.deepcopy(EXPECTED_PROTECTED_POLICY),
"cases": [case],
}
path = tmp_path / "portfolio.json"
@@ -203,6 +206,105 @@ def test_raw_evidence_and_hashes_are_exact_and_recomputable(tmp_path: Path) -> N
evaluate_cases(path, {case["sample_id"]})
def test_portfolio_schema_policy_metadata_and_lineage_are_strict(
tmp_path: Path,
) -> None:
case = detection_case("strict-contract")
portfolio = {
"schema_version": 2,
"portfolio_kind": "synthetic_contract",
"portfolio_id": "synthetic-strict-contract",
"portfolio_lineage": {
"origin": "repository_fixture",
"source_path": "synthetic.json",
"version": "1",
},
"split_roles": ["test"],
"selection_policy": "Fixed before evaluation; no selection.",
"claim_boundary": "Synthetic evaluator test; not product accuracy.",
"protected_policy": copy.deepcopy(EXPECTED_PROTECTED_POLICY),
"cases": [case],
}
path = tmp_path / "strict.json"
def evaluate(value: dict) -> dict:
path.write_text(json.dumps(value, ensure_ascii=False), encoding="utf-8")
return evaluate_cases(path, {case["sample_id"]})
assert evaluate(portfolio)["case_count"] == 1
for invalid_version in (1, True, "2"):
invalid = copy.deepcopy(portfolio)
invalid["schema_version"] = invalid_version
with pytest.raises(
ValueError, match="schema_version must be exactly integer 2"
):
evaluate(invalid)
invalid_policy = copy.deepcopy(portfolio)
invalid_policy["protected_policy"]["test_feedback_allowed"] = True
with pytest.raises(ValueError, match="protected_policy must exactly equal"):
evaluate(invalid_policy)
invalid_metadata = copy.deepcopy(portfolio)
invalid_metadata["cases"][0]["metadata"]["source"] = "unknown"
with pytest.raises(ValueError, match="metadata.source must be a meaningful"):
evaluate(invalid_metadata)
invalid_resolution = copy.deepcopy(portfolio)
invalid_resolution["cases"][0]["metadata"]["resolution_m"] = 0
with pytest.raises(ValueError, match="metadata.resolution_m must be positive"):
evaluate(invalid_resolution)
invalid_lineage = copy.deepcopy(portfolio)
del invalid_lineage["cases"][0]["lineage"]["prediction"]["derivation"]
with pytest.raises(ValueError, match="lineage.prediction missing"):
evaluate(invalid_lineage)
def test_portfolio_kind_separates_synthetic_and_governed_product_claims(
tmp_path: Path,
) -> None:
fixture_path = (
ROOT / "fixtures" / "accuracy" / "p4" / "protected-baseline-cases.json"
)
synthetic = json.loads(fixture_path.read_text(encoding="utf-8"))
allowed = {item["sample_id"] for item in synthetic["cases"]}
path = tmp_path / "portfolio.json"
missing_kind = copy.deepcopy(synthetic)
del missing_kind["portfolio_kind"]
path.write_text(json.dumps(missing_kind), encoding="utf-8")
with pytest.raises(ValueError, match="portfolio_kind"):
evaluate_cases(path, allowed)
confused = copy.deepcopy(synthetic)
confused["claim_boundary"] = "Governed product baseline accuracy evidence."
path.write_text(json.dumps(confused), encoding="utf-8")
with pytest.raises(ValueError, match="synthetic_contract"):
evaluate_cases(path, allowed)
governed = json.loads(
json.dumps(synthetic)
.replace("Synthetic", "Governed")
.replace("synthetic", "governed")
.replace("repository_fixture", "governed_product_evaluation")
)
governed["portfolio_kind"] = "governed_product_baseline"
governed["portfolio_id"] = "governed-product-baseline-test"
governed["claim_boundary"] = (
"Governed product baseline metrics recomputed from protected raw cases; "
"inference provenance is validated separately."
)
path.write_text(json.dumps(governed), encoding="utf-8")
report = evaluate_cases(path, allowed)
assert report["portfolio_kind"] == "governed_product_baseline"
assert set(report["evaluated_task_families"]) == TASKS
governed["cases"] = governed["cases"][:-1]
path.write_text(json.dumps(governed), encoding="utf-8")
def test_ap_ties_use_stable_ids_and_matching_is_class_aware() -> None:
references = [{"id": "r", "class": "building", "bbox": [0, 0, 4, 4]}]
predictions = [
@@ -233,6 +335,83 @@ def test_ap_ties_use_stable_ids_and_matching_is_class_aware() -> None:
assert detection_ap(wrong_class, references, 0.5) == pytest.approx(0.5)
def test_detection_ap_and_calibration_are_pooled_globally_and_per_subgroup(
tmp_path: Path,
) -> None:
first = detection_case("a-case")
first["predictions"] = [
{
"id": "p-true",
"class": "building",
"bbox": [0, 0, 4, 4],
"confidence": 0.9,
}
]
second = detection_case("b-case")
second["predictions"] = [
{
"id": "p-false",
"class": "building",
"bbox": [10, 10, 12, 12],
"confidence": 0.9,
},
{
"id": "p-true",
"class": "building",
"bbox": [0, 0, 4, 4],
"confidence": 0.8,
},
]
portfolio = {
"schema_version": 2,
"portfolio_kind": "synthetic_contract",
"portfolio_id": "synthetic-pooled-detection",
"portfolio_lineage": {
"origin": "repository_fixture",
"source_path": "pooled.json",
"version": "1",
},
"split_roles": ["test"],
"selection_policy": "Fixed before evaluation; no selection.",
"claim_boundary": "Synthetic evaluator test; not product accuracy.",
"protected_policy": copy.deepcopy(EXPECTED_PROTECTED_POLICY),
"cases": [first, second],
}
path = tmp_path / "pooled.json"
path.write_text(json.dumps(portfolio, ensure_ascii=False), encoding="utf-8")
report = evaluate_cases(path, {"a-case", "b-case"})
case_ap = [item["metrics"]["ap50"] for item in report["results"]]
pooled = report["portfolio_metrics"]["object_detection"]["micro"]
expected_pooled = detection_ap(
[
{**item, "id": f"a-case::{item['id']}", "_sample_id": "a-case"}
for item in first["predictions"]
]
+ [
{**item, "id": f"b-case::{item['id']}", "_sample_id": "b-case"}
for item in second["predictions"]
],
[
{**item, "id": f"a-case::{item['id']}", "_sample_id": "a-case"}
for item in first["references"]
]
+ [
{**item, "id": f"b-case::{item['id']}", "_sample_id": "b-case"}
for item in second["references"]
],
0.5,
)
assert pooled["ap50"] == expected_pooled
assert pooled["ap50"] != pytest.approx(sum(case_ap) / len(case_ap))
assert sum(item["count"] for item in pooled["calibration"]["bins"]) == 3
subgroup = report["subgroups"]["dimensions"]["region"]["strata"]["flanders"]
subgroup_calibration = subgroup["task_metrics"]["object_detection"]["micro"][
"calibration"
]
assert sum(item["count"] for item in subgroup_calibration["bins"]) == 3
def test_raster_requires_exact_rectangular_alignment_masks_nodata_and_classes() -> None:
invalid_case = detection_case("invalid-class")
invalid_case["predictions"][0]["class"] = "road"
@@ -270,6 +449,11 @@ def test_raster_requires_exact_rectangular_alignment_masks_nodata_and_classes()
invalid_nodata["predictions"][0][0] = -9999
with pytest.raises(ValueError, match="marks nodata as valid"):
evaluate_raster_classification(invalid_nodata)
singular = raster_case("singular")
for side in ("reference", "prediction"):
singular["raster_context"][side]["transform"] = [1, 2, 0, 2, 4, 0]
with pytest.raises(ValueError, match="affine transform is singular"):
evaluate_raster_classification(singular)
def test_polygon_metrics_require_valid_geometry_projected_crs_and_metres() -> None:
@@ -280,11 +464,59 @@ def test_polygon_metrics_require_valid_geometry_projected_crs_and_metres() -> No
]["mean_iou"]
== 1.0
)
outer = [
[100000, 200000],
[100020, 200000],
[100020, 200020],
[100000, 200020],
[100000, 200000],
]
hole = [
[100005, 200005],
[100010, 200005],
[100010, 200010],
[100005, 200010],
[100005, 200005],
]
polygon_geometry = {"type": "Polygon", "coordinates": [outer, hole]}
geojson_polygon = polygon_case()
for side in ("references", "predictions"):
del geojson_polygon[side][0]["polygon"]
geojson_polygon[side][0]["geometry"] = copy.deepcopy(polygon_geometry)
polygon_result = evaluate_vector_comparison(geojson_polygon)
assert polygon_result["metrics"]["mean_iou"] == 1.0
assert polygon_result["raw"]["references"][0]["geometry"] == polygon_geometry
second = [
[100030, 200000],
[100040, 200000],
[100040, 200010],
[100030, 200010],
[100030, 200000],
]
multipolygon_geometry = {
"type": "MultiPolygon",
"coordinates": [[outer, hole], [second]],
}
geojson_multi = polygon_case()
for side in ("references", "predictions"):
del geojson_multi[side][0]["polygon"]
geojson_multi[side][0]["geometry"] = copy.deepcopy(multipolygon_geometry)
assert evaluate_vector_comparison(geojson_multi)["metrics"]["mean_iou"] == 1.0
geographic = polygon_case()
geographic["spatial_context"]["crs"] = "EPSG:4326"
with pytest.raises(ValueError, match="projected CRS"):
evaluate_vector_comparison(geographic)
mercator = polygon_case()
mercator["spatial_context"]["crs"] = "EPSG:3857"
with pytest.raises(ValueError, match="Mercator is unsuitable"):
evaluate_vector_comparison(mercator)
wrong_geography = polygon_case()
wrong_geography["spatial_context"]["crs"] = "EPSG:32660"
with pytest.raises(ValueError, match="does not overlap"):
evaluate_vector_comparison(wrong_geography)
wrong_units = polygon_case()
wrong_units["spatial_context"]["coordinate_units"] = "degree"
@@ -303,6 +535,61 @@ def test_polygon_metrics_require_valid_geometry_projected_crs_and_metres() -> No
evaluate_vector_comparison(bowtie)
def test_failure_gallery_covers_geometry_raster_calibration_and_contexts() -> None:
segmentation = polygon_case("footprint_segmentation")
segmentation["predictions"][0]["polygon"] = [
[100000, 200000],
[100012, 200000],
[100012, 200010],
[100000, 200010],
[100000, 200000],
]
segmentation_result = evaluate_footprint_segmentation(segmentation)
segmentation_codes = {
item["error_code"] for item in segmentation_result["failures"]
}
assert {"M-BOUNDARY", "M-AREA-BIAS"} <= segmentation_codes
raster = raster_case("raster-taxonomy")
raster["metadata"]["tile_edge"] = True
raster["predictions"][0][0] = 1
raster_result = evaluate_raster_classification(raster)
raster_failure = next(
item
for item in raster_result["failures"]
if item["kind"] == "raster_misclassification"
)
assert raster_failure["error_code"] == "M-CLASS"
assert "tile_edge" in raster_failure["contexts"]
detection = detection_case("context-taxonomy")
detection["references"] = []
detection["predictions"] = [
{
"id": "high-confidence-fp",
"class": "building",
"bbox": [10, 10, 12, 12],
"confidence": 0.95,
}
]
detection["config"]["fixed_diagnostic_risk_thresholds"] = [0.5, 0.9]
detection["metadata"]["tile_edge"] = True
detection["metadata"]["ood"] = True
detection_result = evaluate_object_detection(detection)
false_positive = next(
item
for item in detection_result["failures"]
if item["kind"] == "false_positive"
)
assert {"tile_edge", "high_confidence", "out_of_distribution"} <= set(
false_positive["contexts"]
)
assert {"M-MISCALIBRATED", "M-OOD"} <= set(false_positive["secondary_error_codes"])
assert any(
item["error_code"] == "M-MISCALIBRATED" for item in detection_result["failures"]
)
def test_terrain_rejects_non_finite_and_validation_counts_only_critical_misses() -> (
None
):
@@ -345,26 +632,60 @@ def test_terrain_rejects_non_finite_and_validation_counts_only_critical_misses()
evaluate_validation(validation)["metrics"]["blocker_or_critical_miss_count"]
== 1
)
validation["expected_anomalies"] = [{"code": "D-SEVERITY", "severity": "critical"}]
validation["observed_anomalies"] = [{"code": "D-SEVERITY", "severity": "minor"}]
severity_result = evaluate_validation(validation)
assert severity_result["metrics"]["true_positive"] == 0
assert severity_result["metrics"]["false_positive"] == 1
assert severity_result["metrics"]["false_negative"] == 1
assert severity_result["metrics"]["severity_mismatch_count"] == 1
assert severity_result["metrics"]["blocker_or_critical_miss_count"] == 1
validation["expected_anomalies"] = ["D-NO-SEVERITY"]
with pytest.raises(ValueError, match="include code and severity"):
evaluate_validation(validation)
def _subgroup_result(region: str, tp: int, fp: int, fn: int) -> dict:
def _subgroup_result(region: str, index: int, tp: int, fp: int, fn: int) -> dict:
metadata = copy.deepcopy(METADATA)
metadata["region"] = region
sample_id = f"{region}-{index}"
reference = {"id": "r", "class": "building", "bbox": [0, 0, 1, 1]}
prediction = {
"id": "p",
"class": "building",
"bbox": [0, 0, 1, 1],
"confidence": 0.8,
}
return {
"sample_id": sample_id,
"task": "object_detection",
"metadata": metadata,
"metrics": {**count_metrics(tp, fp, fn), "ap50": 0.5, "ap50_95": 0.4},
"raw": {
"sample_id": sample_id,
"classes": ["building"],
"references": [reference],
"predictions_pre_filter": [prediction],
"predictions_post_filter": [prediction],
"matches": [
{
"prediction_id": "p",
"reference_id": "r",
"overlap": 1.0,
"confidence": 0.8,
"class": "building",
}
],
},
"failures": [],
}
def test_subgroups_report_task_metrics_support_ci_and_worst_stratum() -> None:
results = [
*[_subgroup_result("strong", 10, 0, 0) for _ in range(5)],
*[_subgroup_result("weak", 1, 4, 4) for _ in range(5)],
*[_subgroup_result("strong", index, 10, 0, 0) for index in range(5)],
*[_subgroup_result("weak", index, 1, 4, 4) for index in range(5)],
]
report = subgroup_report(results)
region = report["dimensions"]["region"]
@@ -376,7 +697,7 @@ def test_subgroups_report_task_metrics_support_ci_and_worst_stratum() -> None:
assert weak["macro"]["f1_case_support"] == 5
assert region["worst_stratum_by_task"]["object_detection"]["stratum"] == "weak"
insufficient = subgroup_report([_subgroup_result("thin", 1, 0, 0)])
insufficient = subgroup_report([_subgroup_result("thin", 0, 1, 0, 0)])
thin = insufficient["dimensions"]["region"]["strata"]["thin"]
assert thin["task_metrics"]["object_detection"]["status"] == "insufficient_support"
assert thin["release_gate_status"] == "not_evaluable"
backend/tests/test_accuracy_phase4_split_hardening.py
+335 -19
View File
@@ -1,6 +1,7 @@
from __future__ import annotations
import copy
import hashlib
import json
import sys
from pathlib import Path
@@ -28,13 +29,25 @@ def load_source() -> dict:
return json.loads(SOURCE.read_text(encoding="utf-8"))
def build_fixture_manifests(source: dict) -> tuple[dict, dict, dict]:
return build_manifests(source, trusted_fixture_mode=True)
def assert_fixture_training_inputs_safe(
input_paths: list[Path], input_records: list[dict], protected: dict
) -> None:
assert_training_inputs_safe(
input_paths, input_records, protected, trusted_fixture_mode=True
)
def test_normative_roles_hashes_and_source_order_are_enforced() -> None:
source = load_source()
development, protected, leakage = build_manifests(source)
development, protected, leakage = build_fixture_manifests(source)
reversed_source = copy.deepcopy(source)
reversed_source["samples"].reverse()
reversed_development, reversed_protected, reversed_leakage = build_manifests(
reversed_source
reversed_development, reversed_protected, reversed_leakage = (
build_fixture_manifests(reversed_source)
)
assert leakage["status"] == "pass"
@@ -82,7 +95,7 @@ def test_cross_split_lineage_and_content_collisions_fail(
) -> None:
source = load_source()
source["samples"][8][field] = source["samples"][0][field]
_development, _protected, leakage = build_manifests(source)
_development, _protected, leakage = build_fixture_manifests(source)
assert leakage["status"] == "fail"
assert expected_code in {item["code"] for item in leakage["findings"]}
@@ -98,7 +111,7 @@ def test_cross_split_lineage_and_content_collisions_fail(
def test_near_duplicate_fingerprints_fail(field: str, expected_code: str) -> None:
source = load_source()
source["samples"][8][field] = source["samples"][0][field]
_development, _protected, leakage = build_manifests(source)
_development, _protected, leakage = build_fixture_manifests(source)
assert leakage["status"] == "fail"
assert expected_code in {item["code"] for item in leakage["findings"]}
@@ -111,7 +124,7 @@ def test_object_native_feature_and_spatial_collisions_fail() -> None:
"native_feature_ids"
]
source["samples"][10]["bbox"] = source["samples"][2]["bbox"]
_development, _protected, leakage = build_manifests(source)
_development, _protected, leakage = build_fixture_manifests(source)
codes = {item["code"] for item in leakage["findings"]}
assert {"S-OBJECT-INSTANCE", "S-NATIVE-FEATURE", "S-SPATIAL-OVERLAP"} <= codes
@@ -121,32 +134,34 @@ def test_non_metric_crs_and_missing_normative_role_fail_closed() -> None:
geographic = load_source()
geographic["crs"] = "EPSG:4326"
with pytest.raises(LeakageError, match="projected in metres"):
build_manifests(geographic)
build_fixture_manifests(geographic)
missing = load_source()
missing["samples"] = [
item for item in missing["samples"] if item["split"] != "calibration"
]
with pytest.raises(LeakageError, match="Required splits are absent"):
build_manifests(missing)
build_fixture_manifests(missing)
def test_training_firewall_only_allows_train_and_binds_protected_lineage() -> None:
development, protected, leakage = build_manifests(load_source())
development, protected, leakage = build_fixture_manifests(load_source())
assert leakage["status"] == "pass"
train = [item for item in development["samples"] if item["split"] == "train"]
validation = next(item for item in development["samples"] if item["split"] == "val")
protected_item = protected["samples"][0]
assert_training_inputs_safe([], train, protected)
assert_fixture_training_inputs_safe([], train, protected)
with pytest.raises(LeakageError, match="non_train_role"):
assert_training_inputs_safe([], [validation], protected)
assert_fixture_training_inputs_safe([], [validation], protected)
with pytest.raises(LeakageError, match="protected_identity"):
disguised = copy.deepcopy(train[0])
disguised["source_family"] = protected_item["source_family"]
assert_training_inputs_safe([], [disguised], protected)
assert_fixture_training_inputs_safe([], [disguised], protected)
with pytest.raises(LeakageError, match="protected_path"):
assert_training_inputs_safe([Path("vault/protected/test.json")], [], protected)
assert_fixture_training_inputs_safe(
[Path("vault/protected/test.json")], [], protected
)
def test_failed_generation_writes_status_but_no_consumable_manifests(
@@ -159,12 +174,18 @@ def test_failed_generation_writes_status_but_no_consumable_manifests(
output = tmp_path / "out"
with pytest.raises(LeakageError, match="Leakage gate failed"):
generate(source_path, output)
generate(source_path, output, trusted_fixture_mode=True)
status = json.loads((output / "generation-status.json").read_text(encoding="utf-8"))
assert status["status"] == "fail"
assert not (output / "development-split-manifest.json").exists()
assert not (output / "protected-split-manifest.json").exists()
assert status["consumable_manifests_valid"] is False
for name in (
"development-split-manifest.json",
"protected-split-manifest.json",
):
tombstone = json.loads((output / name).read_text(encoding="utf-8"))
assert tombstone["status"] == "invalidated"
assert tombstone["consumable"] is False
def test_deterministic_grouped_assignment_is_stable_and_keeps_relatives_together() -> (
@@ -196,11 +217,11 @@ def test_deterministic_grouped_assignment_is_stable_and_keeps_relatives_together
item.pop("split")
source["samples"][1]["group_id"] = source["samples"][0]["group_id"]
development, protected, leakage = build_manifests(source)
development, protected, leakage = build_fixture_manifests(source)
reversed_source = copy.deepcopy(source)
reversed_source["samples"].reverse()
reversed_development, reversed_protected, reversed_leakage = build_manifests(
reversed_source
reversed_development, reversed_protected, reversed_leakage = (
build_fixture_manifests(reversed_source)
)
assigned = {
@@ -220,3 +241,298 @@ def test_deterministic_grouped_assignment_is_stable_and_keeps_relatives_together
assert reversed_development["manifest_sha256"] == development["manifest_sha256"]
assert reversed_protected["manifest_sha256"] == protected["manifest_sha256"]
assert reversed_leakage == leakage
def test_source_cannot_weaken_mandatory_roles_or_policy_floors() -> None:
source = load_source()
source["required_splits"] = ["train", "val", "test"]
with pytest.raises(LeakageError, match="mandatory role order"):
build_fixture_manifests(source)
grouped = load_source()
grouped["assignment_mode"] = "deterministic_grouped"
grouped["split_assignment"] = {"roles": ["train", "val"]}
for item in grouped["samples"]:
item.pop("split")
with pytest.raises(LeakageError, match="mandatory role order"):
build_fixture_manifests(grouped)
for field, value in (
("independence_buffer_m", 1999),
("perceptual_hamming_threshold", 3),
("label_geometry_hamming_threshold", 1),
):
weakened = load_source()
weakened[field] = value
with pytest.raises(LeakageError, match="code-owned minimum"):
build_fixture_manifests(weakened)
def test_task_coverage_gap_and_even_justified_exemption_fail_honestly() -> None:
source = load_source()
source["samples"] = [
item
for item in source["samples"]
if item["sample_id"] != "validation-test-national"
]
_development, _protected, leakage = build_fixture_manifests(source)
assert leakage["status"] == "fail"
assert "S-PROTECTED-TASK-COVERAGE-MISSING" in {
finding["code"] for finding in leakage["findings"]
}
source["protected_task_exemptions"] = {
"geospatial_data_validation": (
"No evaluator-visible reference exists; challenge data remains sealed."
)
}
_development, _protected, leakage = build_fixture_manifests(source)
assert leakage["status"] == "fail"
assert "S-PROTECTED-TASK-COVERAGE-EXEMPTED" in {
finding["code"] for finding in leakage["findings"]
}
def test_identifiers_and_acquisition_dates_are_canonical_leakage_keys() -> None:
source = load_source()
source["samples"][8]["group_id"] = " G01 "
_development, _protected, leakage = build_fixture_manifests(source)
assert "S-SPATIAL-GROUP" in {item["code"] for item in leakage["findings"]}
temporal = load_source()
temporal["samples"][8]["acquisition_date"] = temporal["samples"][0][
"acquisition_date"
]
_development, _protected, leakage = build_fixture_manifests(temporal)
assert "S-ACQUISITION-DATE" in {item["code"] for item in leakage["findings"]}
ambiguous = load_source()
ambiguous["samples"][8]["sample_id"] = " DET-TRAIN-A "
with pytest.raises(LeakageError, match="ambiguous canonical sample_id"):
build_fixture_manifests(ambiguous)
def test_challenge_is_sealed_in_standard_manifest() -> None:
_development, protected, leakage = build_fixture_manifests(load_source())
assert leakage["status"] == "pass"
challenge = [item for item in protected["samples"] if item["split"] == "challenge"]
forbidden = {
"label_sha256",
"label_geometry_hash",
"label_geometry_fingerprint",
"object_ids",
"native_feature_ids",
"record_sha256",
"label_path",
"label_geometry_path",
}
assert challenge
assert all(item["sealed"] is True for item in challenge)
assert all(not (forbidden & set(item)) for item in challenge)
def test_firewall_rejects_empty_tampered_wrong_and_renamed_manifests(
tmp_path: Path,
) -> None:
development, protected, leakage = build_fixture_manifests(load_source())
assert leakage["status"] == "pass"
train = [item for item in development["samples"] if item["split"] == "train"]
with pytest.raises(LeakageError, match="empty manifest"):
assert_fixture_training_inputs_safe([], train, {})
with pytest.raises(LeakageError, match="missing fields"):
assert_fixture_training_inputs_safe([], train, development)
tampered = copy.deepcopy(protected)
tampered["samples"].pop()
with pytest.raises(LeakageError, match="checksum mismatch"):
assert_fixture_training_inputs_safe([], train, tampered)
renamed = tmp_path / "ordinary-training-input.json"
renamed.write_text(json.dumps(protected), encoding="utf-8")
with pytest.raises(LeakageError, match="protected_manifest_content"):
assert_fixture_training_inputs_safe([renamed], train, protected)
def _canonical_json_sha256(value: object) -> str:
return hashlib.sha256(
json.dumps(
value, ensure_ascii=False, sort_keys=True, separators=(",", ":")
).encode("utf-8")
).hexdigest()
def make_governed_source(tmp_path: Path) -> dict:
source = load_source()
source["dataset_version"] = "governed-production-v1"
source["claim_boundary"] = "Governed production split source."
p3_items: list[dict] = []
provenance_records: list[dict] = []
asset_fields = {
"raw_image": ("raw_image_path", "raw_image_sha256"),
"processed_image": ("processed_image_path", "processed_image_sha256"),
"label": ("label_path", "label_sha256"),
"label_geometry": ("label_geometry_path", "label_geometry_hash"),
}
for sample in source["samples"]:
assets: dict[str, dict] = {}
p3_ids: dict[str, str] = {}
for role, (path_field, hash_field) in asset_fields.items():
relative = Path("assets") / sample["sample_id"] / f"{role}.bin"
path = tmp_path / relative
path.parent.mkdir(parents=True, exist_ok=True)
path.write_bytes(f"{sample['sample_id']}:{role}:governed".encode())
digest = hashlib.sha256(path.read_bytes()).hexdigest()
p3_id = hashlib.sha256(
f"{sample['sample_id']}:{role}".encode()
).hexdigest()[:20]
relative_posix = relative.as_posix()
sample[path_field] = relative_posix
sample[hash_field] = digest
p3_ids[role] = p3_id
assets[role] = {
"path": relative_posix,
"sha256": digest,
"size_bytes": path.stat().st_size,
"p3_item_id": p3_id,
}
p3_items.append(
{
"item_id": p3_id,
"path": relative_posix,
"sha256": digest,
"size_bytes": path.stat().st_size,
"status": "examined",
"read_status": "readable",
"recommended_action": "accept",
"empty_content": False,
"schema_conformity": "conformant",
"anomalies": [],
}
)
provenance_id = (
"prov-" + hashlib.sha256(sample["sample_id"].encode()).hexdigest()[:24]
)
sample["governance_binding"] = {
"source_provenance_record_id": provenance_id,
"p3_item_ids": p3_ids,
}
provenance_records.append(
{
"record_id": provenance_id,
"sample_id": sample["sample_id"],
"status": "accepted",
"lineage_status": "complete",
"training_allowed": True,
"perceptual_image_hash": sample["perceptual_image_hash"],
"label_geometry_fingerprint": sample["label_geometry_fingerprint"],
"assets": assets,
}
)
p3 = {
"schema_version": 1,
"scan_id": "p3-test-governed",
"scanner_version": "3.0.3",
"completed_at": "2026-08-02T12:00:00+02:00",
"items": p3_items,
"reconciliation": {
"examined": len(p3_items),
"skipped": 0,
"unreachable": 0,
"inventory_total": len(p3_items),
"reconciles": True,
},
}
provenance = {
"schema_version": 1,
"manifest_type": "geointel_phase4_source_provenance",
"status": "pass",
"records": provenance_records,
"records_canonical_json_sha256": _canonical_json_sha256(provenance_records),
}
p3_path = tmp_path / "p3.json"
provenance_path = tmp_path / "provenance.json"
p3_path.write_text(json.dumps(p3), encoding="utf-8")
provenance_path.write_text(json.dumps(provenance), encoding="utf-8")
source["governance_evidence"] = {
"p3_scan_manifest": {
"path": p3_path.name,
"sha256": hashlib.sha256(p3_path.read_bytes()).hexdigest(),
},
"source_provenance_manifest": {
"path": provenance_path.name,
"sha256": hashlib.sha256(provenance_path.read_bytes()).hexdigest(),
},
}
return source
def test_fixture_mode_is_explicit_and_source_metadata_cannot_enable_it() -> None:
with pytest.raises(LeakageError, match="explicit trusted_fixture_mode"):
build_manifests(load_source())
source = load_source()
source["trusted_fixture_mode"] = True
with pytest.raises(LeakageError, match="cannot be enabled by source metadata"):
build_manifests(source, trusted_fixture_mode=True)
def test_empty_or_arbitrary_governance_json_is_rejected(tmp_path: Path) -> None:
source = load_source()
source["dataset_version"] = "governed-production-v1"
source["claim_boundary"] = "Governed production split source."
p3 = tmp_path / "p3.json"
provenance = tmp_path / "provenance.json"
p3.write_text("{}", encoding="utf-8")
provenance.write_text("{}", encoding="utf-8")
source["governance_evidence"] = {
"p3_scan_manifest": {
"path": p3.name,
"sha256": hashlib.sha256(p3.read_bytes()).hexdigest(),
},
"source_provenance_manifest": {
"path": provenance.name,
"sha256": hashlib.sha256(provenance.read_bytes()).hexdigest(),
},
}
with pytest.raises(LeakageError, match="non-empty JSON object"):
build_manifests(source, source_root=tmp_path)
def test_governed_records_require_exact_provenance_paths_and_live_bytes(
tmp_path: Path,
) -> None:
source = make_governed_source(tmp_path)
development, protected, leakage = build_manifests(source, source_root=tmp_path)
assert leakage["status"] == "pass"
assert protected["source_trust"]["production_accuracy_use_allowed"] is True
train = [item for item in development["samples"] if item["split"] == "train"]
assert_training_inputs_safe([], train, protected)
no_paths = copy.deepcopy(train[0])
no_paths.pop("content_path_bindings")
with pytest.raises(LeakageError, match="missing_accessible_content_paths"):
assert_training_inputs_safe([], [no_paths], protected)
relabeled = copy.deepcopy(
next(item for item in protected["samples"] if item["split"] == "test")
)
relabeled["split"] = "train"
relabeled.pop("content_path_bindings")
for field in ("sample_id", "group_id", "source_family", "temporal_family"):
relabeled[field] = f"spoofed-{field}"
with pytest.raises(
LeakageError, match="unavailable_provenance_record|protected_identity"
):
assert_training_inputs_safe([], [relabeled], protected)
broken_binding = copy.deepcopy(source)
broken_binding["samples"][0]["governance_binding"]["p3_item_ids"]["raw_image"] = (
"0" * 20
)
with pytest.raises(LeakageError, match="provenance binding mismatch|P3 record"):
build_manifests(broken_binding, source_root=tmp_path)
raw_path = Path(train[0]["content_path_bindings"]["raw_image"]["resolved_path"])
raw_path.write_bytes(b"mutated after manifest creation")
with pytest.raises(LeakageError, match="record_path_hash_binding_mismatch"):
assert_training_inputs_safe([], train, protected)