feat(accuracy): build hardened phase 4 evaluation harness
This commit is contained in:
Jens
@@ -0,0 +1,240 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
import hashlib
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
SCRIPTS = ROOT / "scripts"
|
||||
if str(SCRIPTS) not in sys.path:
|
||||
sys.path.insert(0, str(SCRIPTS))
|
||||
|
||||
from accuracy_phase4_evaluator import evaluate_cases # noqa: E402
|
||||
from generate_accuracy_phase4_splits import ( # noqa: E402
|
||||
LeakageError,
|
||||
assert_training_inputs_safe,
|
||||
build_manifests,
|
||||
)
|
||||
from run_accuracy_phase4_benchmark import ( # noqa: E402
|
||||
EvidenceConflictError,
|
||||
build_release_gate_report,
|
||||
canonical_golden_baseline,
|
||||
firewall_contract_checks,
|
||||
run_workflow,
|
||||
)
|
||||
|
||||
|
||||
SOURCE = ROOT / "fixtures/accuracy/p4/split-source-manifest.json"
|
||||
CASES = ROOT / "fixtures/accuracy/p4/protected-baseline-cases.json"
|
||||
|
||||
|
||||
def load_source() -> dict:
|
||||
return json.loads(SOURCE.read_text(encoding="utf-8"))
|
||||
|
||||
|
||||
def evaluation_inputs() -> tuple[dict, dict, dict, dict]:
|
||||
source = load_source()
|
||||
development, protected, leakage = build_manifests(source)
|
||||
split_result = {
|
||||
"development": development,
|
||||
"protected": protected,
|
||||
"leakage": leakage,
|
||||
}
|
||||
allowed = {
|
||||
item["sample_id"]
|
||||
for item in protected["samples"]
|
||||
if item["split"] in {"test", "background-test"}
|
||||
}
|
||||
evaluation = evaluate_cases(CASES, allowed)
|
||||
portfolio = json.loads(CASES.read_text(encoding="utf-8"))
|
||||
firewall = firewall_contract_checks(split_result, CASES)
|
||||
return split_result, evaluation, portfolio, firewall
|
||||
|
||||
|
||||
def test_split_fixture_is_order_independent_and_has_all_roles() -> None:
|
||||
source = load_source()
|
||||
development, protected, leakage = build_manifests(source)
|
||||
reversed_source = copy.deepcopy(source)
|
||||
reversed_source["samples"].reverse()
|
||||
reversed_development, reversed_protected, reversed_leakage = build_manifests(
|
||||
reversed_source
|
||||
)
|
||||
|
||||
assert leakage["status"] == "pass"
|
||||
assert leakage["finding_count"] == 0
|
||||
assert leakage["split_counts"] == {
|
||||
"background-test": 2,
|
||||
"calibration": 2,
|
||||
"challenge": 4,
|
||||
"test": 7,
|
||||
"train": 3,
|
||||
"val": 3,
|
||||
}
|
||||
assert reversed_development["manifest_sha256"] == development["manifest_sha256"]
|
||||
assert reversed_protected["manifest_sha256"] == protected["manifest_sha256"]
|
||||
assert reversed_leakage == leakage
|
||||
|
||||
|
||||
def test_training_firewall_rejects_non_train_and_protected_lineage() -> None:
|
||||
development, protected, leakage = build_manifests(load_source())
|
||||
assert leakage["status"] == "pass"
|
||||
train = [item for item in development["samples"] if item["split"] == "train"]
|
||||
validation = next(item for item in development["samples"] if item["split"] == "val")
|
||||
|
||||
assert_training_inputs_safe([], train, protected)
|
||||
with pytest.raises(LeakageError, match="non_train_role"):
|
||||
assert_training_inputs_safe([], [validation], protected)
|
||||
with pytest.raises(LeakageError, match="protected_identity"):
|
||||
disguised = dict(train[0])
|
||||
disguised["source_family"] = protected["samples"][0]["source_family"]
|
||||
assert_training_inputs_safe([], [disguised], protected)
|
||||
with pytest.raises(LeakageError, match="protected_path"):
|
||||
assert_training_inputs_safe([CASES], [], protected)
|
||||
|
||||
|
||||
def test_task_evaluator_retains_exact_raw_inputs_metrics_and_failures() -> None:
|
||||
_split_result, report, _portfolio, _firewall = evaluation_inputs()
|
||||
|
||||
assert report["task_count"] == 7
|
||||
assert report["case_count"] == 9
|
||||
assert len(report["task_inventory"]) >= 15
|
||||
assert len(report["failures"]) == 11
|
||||
assert report["subgroups"]["overall_status"] == "not_evaluable"
|
||||
assert all(
|
||||
{
|
||||
"references",
|
||||
"predictions_pre_filter",
|
||||
"predictions_post_filter",
|
||||
"config",
|
||||
"input_lineage",
|
||||
"portfolio_lineage",
|
||||
}
|
||||
<= set(item["raw"])
|
||||
for item in report["results"]
|
||||
)
|
||||
detection = next(
|
||||
item
|
||||
for item in report["results"]
|
||||
if item["sample_id"] == "det-test-flanders-urban"
|
||||
)
|
||||
assert len(detection["raw"]["predictions_pre_filter"]) == 4
|
||||
assert len(detection["raw"]["predictions_post_filter"]) == 3
|
||||
assert detection["metrics"]["true_positive"] == 2
|
||||
assert detection["metrics"]["false_positive"] == 1
|
||||
assert detection["metrics"]["ap50"] is not None
|
||||
empty = next(
|
||||
item
|
||||
for item in report["results"]
|
||||
if item["sample_id"] == "background-test-pure-empty"
|
||||
)
|
||||
assert empty["metrics"]["precision"] is None
|
||||
assert empty["metrics"]["recall"] is None
|
||||
assert empty["metrics"]["f1"] is None
|
||||
|
||||
|
||||
def test_product_prerequisites_cannot_pass_without_executed_baseline() -> None:
|
||||
split_result, evaluation, portfolio, firewall = evaluation_inputs()
|
||||
product_gates = {
|
||||
"active_model_available_and_hash_verified": {"status": "pass"},
|
||||
"authoritative_reference_portfolio_available": {"status": "pass"},
|
||||
"human_review_complete": {"status": "pass"},
|
||||
"split_independence": {"status": "pass"},
|
||||
"phase3_leakage_resolved": {"status": "pass"},
|
||||
"protected_storage_isolation": {"status": "pass"},
|
||||
"executed_product_incumbent_baseline": {
|
||||
"status": "not_evaluable",
|
||||
"reason": "no raw active-model inference",
|
||||
},
|
||||
"representative_product_subgroup_support": {"status": "pass"},
|
||||
}
|
||||
report = build_release_gate_report(
|
||||
split_result,
|
||||
evaluation,
|
||||
portfolio,
|
||||
canonical_golden_baseline(),
|
||||
firewall,
|
||||
product_gates,
|
||||
)
|
||||
|
||||
assert report["local_harness_status"] == "pass"
|
||||
assert report["product_benchmark_status"] == "not_evaluable"
|
||||
assert report["status"] == "not_evaluable"
|
||||
assert report["phase_decision"] == "blocked"
|
||||
|
||||
|
||||
def test_one_workflow_is_byte_reproducible_complete_and_fail_closed(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
output = tmp_path / "p4"
|
||||
first = run_workflow(ROOT, output)
|
||||
first_bytes = {path.name: path.read_bytes() for path in output.glob("*.json")}
|
||||
second = run_workflow(ROOT, output)
|
||||
second_bytes = {path.name: path.read_bytes() for path in output.glob("*.json")}
|
||||
|
||||
assert first == second
|
||||
assert first_bytes == second_bytes
|
||||
assert first["local_harness_status"] == "pass"
|
||||
assert first["product_benchmark_status"] == "not_evaluable"
|
||||
assert first["status"] == "not_evaluable"
|
||||
assert first["phase4_done"] is False
|
||||
assert first["phase5_ready"] is False
|
||||
required = {
|
||||
"acceptance-gates.json",
|
||||
"aoi-metrics.json",
|
||||
"baseline-raw-predictions.json",
|
||||
"benchmark-manifest.json",
|
||||
"calibration-metrics.json",
|
||||
"candidate-vs-incumbent.json",
|
||||
"development-split-manifest.json",
|
||||
"error-taxonomy.json",
|
||||
"evaluation-contract.json",
|
||||
"failure-gallery.json",
|
||||
"generation-status.json",
|
||||
"human-review-summary.json",
|
||||
"input-manifest.json",
|
||||
"latency-and-reliability.json",
|
||||
"leakage-gate-report.json",
|
||||
"metric-report.json",
|
||||
"object-metrics.json",
|
||||
"protected-split-manifest.json",
|
||||
"reference-implementation-baseline.json",
|
||||
"release-gate-report.json",
|
||||
"split-and-leakage-audit.json",
|
||||
"stratified-metrics.json",
|
||||
"tile-metrics.json",
|
||||
"workflow-summary.json",
|
||||
"evidence-manifest.json",
|
||||
}
|
||||
assert required == set(first_bytes)
|
||||
manifest = json.loads(
|
||||
(output / "evidence-manifest.json").read_text(encoding="utf-8")
|
||||
)
|
||||
assert manifest["artifact_count"] == len(required) - 1
|
||||
for item in manifest["artifacts"]:
|
||||
path = output / item["path"]
|
||||
assert path.stat().st_size == item["size_bytes"]
|
||||
assert hashlib.sha256(path.read_bytes()).hexdigest() == item["sha256"]
|
||||
gates = json.loads(
|
||||
(output / "release-gate-report.json").read_text(encoding="utf-8")
|
||||
)
|
||||
assert gates["promotion_allowed"] is False
|
||||
assert all(item["status"] == "pass" for item in gates["local_gates"].values())
|
||||
assert {item["status"] for item in gates["product_gates"].values()} <= {
|
||||
"pass",
|
||||
"fail",
|
||||
"not_evaluable",
|
||||
}
|
||||
|
||||
|
||||
def test_immutable_workflow_refuses_to_replace_changed_evidence(tmp_path: Path) -> None:
|
||||
output = tmp_path / "p4"
|
||||
run_workflow(ROOT, output)
|
||||
(output / "workflow-summary.json").write_text("{}\n", encoding="utf-8")
|
||||
|
||||
with pytest.raises(EvidenceConflictError, match="Refusing to overwrite"):
|
||||
run_workflow(ROOT, output)
|
||||
@@ -0,0 +1,397 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
import hashlib
|
||||
import json
|
||||
import math
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
SCRIPTS = ROOT / "scripts"
|
||||
if str(SCRIPTS) not in sys.path:
|
||||
sys.path.insert(0, str(SCRIPTS))
|
||||
|
||||
from accuracy_phase4_evaluator import ( # noqa: E402
|
||||
TASKS,
|
||||
canonical_hash,
|
||||
count_metrics,
|
||||
detection_ap,
|
||||
evaluate_cases,
|
||||
evaluate_object_detection,
|
||||
evaluate_footprint_segmentation,
|
||||
evaluate_raster_classification,
|
||||
evaluate_terrain,
|
||||
evaluate_validation,
|
||||
evaluate_vector_comparison,
|
||||
subgroup_report,
|
||||
task_inventory,
|
||||
)
|
||||
|
||||
|
||||
METADATA = {
|
||||
"region": "flanders",
|
||||
"municipality": "Mol",
|
||||
"urbanity": "urban",
|
||||
"object_size": "medium",
|
||||
"source": "synthetic-source",
|
||||
"sensor": "synthetic-sensor",
|
||||
"resolution_m": 0.25,
|
||||
"season": "summer",
|
||||
"date": "2026-01-01",
|
||||
"vegetation": "partial",
|
||||
"occlusion": "none",
|
||||
"difficulty": "normal",
|
||||
}
|
||||
|
||||
|
||||
def lineage(sample_id: str) -> dict:
|
||||
return {
|
||||
"reference": {
|
||||
"source_id": f"synthetic:{sample_id}:reference",
|
||||
"source_version": "1",
|
||||
"derivation": "hand_authored_contract_fixture",
|
||||
},
|
||||
"prediction": {
|
||||
"source_id": f"synthetic:{sample_id}:prediction",
|
||||
"source_version": "1",
|
||||
"derivation": "hand_authored_fixed_output",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def detection_case(sample_id: str = "det-1") -> dict:
|
||||
return {
|
||||
"sample_id": sample_id,
|
||||
"task": "object_detection",
|
||||
"split": "test",
|
||||
"metadata": copy.deepcopy(METADATA),
|
||||
"config": {"confidence_threshold": 0.5, "match_iou": 0.5},
|
||||
"lineage": lineage(sample_id),
|
||||
"classes": ["building", "tank"],
|
||||
"references": [{"id": "r-building", "class": "building", "bbox": [0, 0, 4, 4]}],
|
||||
"predictions": [
|
||||
{
|
||||
"id": "p-building",
|
||||
"class": "building",
|
||||
"bbox": [0, 0, 4, 4],
|
||||
"confidence": 0.8,
|
||||
},
|
||||
{
|
||||
"id": "p-filtered",
|
||||
"class": "building",
|
||||
"bbox": [10, 10, 12, 12],
|
||||
"confidence": 0.2,
|
||||
},
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def raster_case(sample_id: str = "raster-1") -> dict:
|
||||
reference_side = {
|
||||
"crs": "EPSG:31370",
|
||||
"transform": [1, 0, 100000, 0, -1, 200000],
|
||||
"shape": [2, 2],
|
||||
"nodata": -9999,
|
||||
"mask": [[True, True], [True, True]],
|
||||
}
|
||||
return {
|
||||
"sample_id": sample_id,
|
||||
"task": "raster_classification",
|
||||
"split": "test",
|
||||
"metadata": copy.deepcopy(METADATA),
|
||||
"config": {},
|
||||
"lineage": lineage(sample_id),
|
||||
"classes": [0, 1],
|
||||
"references": [[0, 1], [1, 0]],
|
||||
"predictions": [[0, 1], [1, 0]],
|
||||
"raster_context": {
|
||||
"reference": reference_side,
|
||||
"prediction": copy.deepcopy(reference_side),
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def polygon_case(task: str = "vector_comparison") -> dict:
|
||||
sample_id = f"{task}-1"
|
||||
config = {"match_iou": 0.5}
|
||||
if task == "footprint_segmentation":
|
||||
config["boundary_tolerance_m"] = 1.0
|
||||
polygon = [
|
||||
[100000, 200000],
|
||||
[100010, 200000],
|
||||
[100010, 200010],
|
||||
[100000, 200010],
|
||||
[100000, 200000],
|
||||
]
|
||||
return {
|
||||
"sample_id": sample_id,
|
||||
"task": task,
|
||||
"split": "test",
|
||||
"metadata": copy.deepcopy(METADATA),
|
||||
"config": config,
|
||||
"lineage": lineage(sample_id),
|
||||
"classes": ["building"],
|
||||
"spatial_context": {
|
||||
"crs": "EPSG:31370",
|
||||
"coordinate_units": "m",
|
||||
"metric": True,
|
||||
},
|
||||
"references": [{"id": "reference", "class": "building", "polygon": polygon}],
|
||||
"predictions": [{"id": "prediction", "class": "building", "polygon": polygon}],
|
||||
}
|
||||
|
||||
|
||||
def test_raw_evidence_and_hashes_are_exact_and_recomputable(tmp_path: Path) -> None:
|
||||
case = detection_case()
|
||||
portfolio = {
|
||||
"schema_version": 2,
|
||||
"portfolio_id": "synthetic-hardening-test",
|
||||
"portfolio_lineage": {
|
||||
"origin": "repository_fixture",
|
||||
"source_path": "synthetic.json",
|
||||
"version": "1",
|
||||
},
|
||||
"split_roles": ["test"],
|
||||
"selection_policy": "Fixed before evaluation; no selection.",
|
||||
"claim_boundary": "Synthetic evaluator test; not product accuracy.",
|
||||
"protected_policy": {"threshold_selection_allowed": False},
|
||||
"cases": [case],
|
||||
}
|
||||
path = tmp_path / "portfolio.json"
|
||||
path.write_text(
|
||||
json.dumps(portfolio, ensure_ascii=False, indent=2) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
report = evaluate_cases(path, {case["sample_id"]})
|
||||
raw = report["results"][0]["raw"]
|
||||
|
||||
assert raw["references"] == case["references"]
|
||||
assert raw["predictions_pre_filter"] == case["predictions"]
|
||||
assert raw["predictions_post_filter"] == case["predictions"][:1]
|
||||
assert raw["config"] == case["config"]
|
||||
assert raw["split"] == "test"
|
||||
assert raw["input_lineage"] == case["lineage"]
|
||||
assert raw["portfolio_lineage"]["declared"] == portfolio["portfolio_lineage"]
|
||||
assert raw["hashes"]["case_input_canonical_json_sha256"] == canonical_hash(case)
|
||||
assert raw["hashes"]["references_canonical_json_sha256"] == canonical_hash(
|
||||
case["references"]
|
||||
)
|
||||
assert (
|
||||
report["portfolio_file_sha256"] == hashlib.sha256(path.read_bytes()).hexdigest()
|
||||
)
|
||||
assert report["portfolio_canonical_json_sha256"] == canonical_hash(portfolio)
|
||||
assert report["results_canonical_json_sha256"] == canonical_hash(report["results"])
|
||||
high_threshold = next(
|
||||
row
|
||||
for row in report["results"][0]["metrics"]["coverage_risk"]
|
||||
if row["threshold"] == 0.9
|
||||
)
|
||||
assert high_threshold["retained_prediction_coverage"] == 0.0
|
||||
assert high_threshold["reference_coverage"] == 0.0
|
||||
assert high_threshold["false_negative_count"] == 1
|
||||
assert high_threshold["risk"] == 1.0
|
||||
|
||||
challenge_exposed = copy.deepcopy(portfolio)
|
||||
challenge_exposed["challenge_labels"] = []
|
||||
path.write_text(json.dumps(challenge_exposed, ensure_ascii=False), encoding="utf-8")
|
||||
with pytest.raises(ValueError, match="Challenge cases and labels"):
|
||||
evaluate_cases(path, {case["sample_id"]})
|
||||
|
||||
|
||||
def test_ap_ties_use_stable_ids_and_matching_is_class_aware() -> None:
|
||||
references = [{"id": "r", "class": "building", "bbox": [0, 0, 4, 4]}]
|
||||
predictions = [
|
||||
{
|
||||
"id": "z-true",
|
||||
"class": "building",
|
||||
"bbox": [0, 0, 4, 4],
|
||||
"confidence": 0.8,
|
||||
},
|
||||
{
|
||||
"id": "a-false",
|
||||
"class": "building",
|
||||
"bbox": [10, 10, 12, 12],
|
||||
"confidence": 0.8,
|
||||
},
|
||||
]
|
||||
forward = detection_ap(predictions, references, 0.5)
|
||||
reverse = detection_ap(list(reversed(predictions)), references, 0.5)
|
||||
assert forward == reverse == pytest.approx(0.5)
|
||||
|
||||
wrong_class = copy.deepcopy(predictions)
|
||||
wrong_class[1] = {
|
||||
"id": "a-tank",
|
||||
"class": "tank",
|
||||
"bbox": [0, 0, 4, 4],
|
||||
"confidence": 0.95,
|
||||
}
|
||||
assert detection_ap(wrong_class, references, 0.5) == pytest.approx(0.5)
|
||||
|
||||
|
||||
def test_raster_requires_exact_rectangular_alignment_masks_nodata_and_classes() -> None:
|
||||
invalid_case = detection_case("invalid-class")
|
||||
invalid_case["predictions"][0]["class"] = "road"
|
||||
with pytest.raises(ValueError, match="outside the declared ontology"):
|
||||
evaluate_object_detection(invalid_case)
|
||||
|
||||
valid = raster_case()
|
||||
valid["predictions"][0][1] = -9999
|
||||
valid["raster_context"]["prediction"]["mask"][0][1] = False
|
||||
result = evaluate_raster_classification(valid)
|
||||
assert result["metrics"]["prediction_coverage"] == pytest.approx(0.75)
|
||||
assert result["metrics"]["per_class"]["1"]["false_negative"] == 1
|
||||
|
||||
jagged = raster_case("jagged")
|
||||
jagged["predictions"][1].pop()
|
||||
with pytest.raises(ValueError, match="exactly rectangular"):
|
||||
evaluate_raster_classification(jagged)
|
||||
|
||||
missing_metadata = raster_case("missing-metadata")
|
||||
del missing_metadata["raster_context"]["prediction"]["crs"]
|
||||
with pytest.raises(ValueError, match="missing"):
|
||||
evaluate_raster_classification(missing_metadata)
|
||||
|
||||
shifted = raster_case("shifted")
|
||||
shifted["raster_context"]["prediction"]["transform"][2] += 1
|
||||
with pytest.raises(ValueError, match="affine alignment differs"):
|
||||
evaluate_raster_classification(shifted)
|
||||
|
||||
invalid_class = raster_case("invalid-class")
|
||||
invalid_class["predictions"][0][0] = 3
|
||||
with pytest.raises(ValueError, match="prediction class outside ontology"):
|
||||
evaluate_raster_classification(invalid_class)
|
||||
|
||||
invalid_nodata = raster_case("invalid-nodata")
|
||||
invalid_nodata["predictions"][0][0] = -9999
|
||||
with pytest.raises(ValueError, match="marks nodata as valid"):
|
||||
evaluate_raster_classification(invalid_nodata)
|
||||
|
||||
|
||||
def test_polygon_metrics_require_valid_geometry_projected_crs_and_metres() -> None:
|
||||
assert evaluate_vector_comparison(polygon_case())["metrics"]["f1"] == 1.0
|
||||
assert (
|
||||
evaluate_footprint_segmentation(polygon_case("footprint_segmentation"))[
|
||||
"metrics"
|
||||
]["mean_iou"]
|
||||
== 1.0
|
||||
)
|
||||
|
||||
geographic = polygon_case()
|
||||
geographic["spatial_context"]["crs"] = "EPSG:4326"
|
||||
with pytest.raises(ValueError, match="projected CRS"):
|
||||
evaluate_vector_comparison(geographic)
|
||||
|
||||
wrong_units = polygon_case()
|
||||
wrong_units["spatial_context"]["coordinate_units"] = "degree"
|
||||
with pytest.raises(ValueError, match="must be 'm'"):
|
||||
evaluate_vector_comparison(wrong_units)
|
||||
|
||||
bowtie = polygon_case()
|
||||
bowtie["predictions"][0]["polygon"] = [
|
||||
[100000, 200000],
|
||||
[100010, 200010],
|
||||
[100010, 200000],
|
||||
[100000, 200010],
|
||||
[100000, 200000],
|
||||
]
|
||||
with pytest.raises(ValueError, match="positive-area and valid"):
|
||||
evaluate_vector_comparison(bowtie)
|
||||
|
||||
|
||||
def test_terrain_rejects_non_finite_and_validation_counts_only_critical_misses() -> (
|
||||
None
|
||||
):
|
||||
terrain = {
|
||||
"sample_id": "terrain",
|
||||
"task": "terrain_interpretation",
|
||||
"split": "test",
|
||||
"metadata": copy.deepcopy(METADATA),
|
||||
"config": {},
|
||||
"lineage": lineage("terrain"),
|
||||
"units": "m_TAW",
|
||||
"references": [1.0, 2.0],
|
||||
"predictions": [1.1, None],
|
||||
}
|
||||
assert evaluate_terrain(terrain)["metrics"]["coverage"] == 0.5
|
||||
for field, value in (("references", math.nan), ("predictions", math.inf)):
|
||||
invalid = copy.deepcopy(terrain)
|
||||
invalid[field][0] = value
|
||||
with pytest.raises(ValueError, match="finite number"):
|
||||
evaluate_terrain(invalid)
|
||||
|
||||
validation = {
|
||||
"sample_id": "validation",
|
||||
"task": "geospatial_data_validation",
|
||||
"split": "test",
|
||||
"metadata": copy.deepcopy(METADATA),
|
||||
"config": {},
|
||||
"lineage": lineage("validation"),
|
||||
"expected_anomalies": [{"code": "D-MAJOR", "severity": "major"}],
|
||||
"observed_anomalies": [],
|
||||
}
|
||||
assert (
|
||||
evaluate_validation(validation)["metrics"]["blocker_or_critical_miss_count"]
|
||||
== 0
|
||||
)
|
||||
validation["expected_anomalies"].append(
|
||||
{"code": "D-CRITICAL", "severity": "critical"}
|
||||
)
|
||||
assert (
|
||||
evaluate_validation(validation)["metrics"]["blocker_or_critical_miss_count"]
|
||||
== 1
|
||||
)
|
||||
validation["expected_anomalies"] = ["D-NO-SEVERITY"]
|
||||
with pytest.raises(ValueError, match="include code and severity"):
|
||||
evaluate_validation(validation)
|
||||
|
||||
|
||||
def _subgroup_result(region: str, tp: int, fp: int, fn: int) -> dict:
|
||||
metadata = copy.deepcopy(METADATA)
|
||||
metadata["region"] = region
|
||||
return {
|
||||
"task": "object_detection",
|
||||
"metadata": metadata,
|
||||
"metrics": {**count_metrics(tp, fp, fn), "ap50": 0.5, "ap50_95": 0.4},
|
||||
"failures": [],
|
||||
}
|
||||
|
||||
|
||||
def test_subgroups_report_task_metrics_support_ci_and_worst_stratum() -> None:
|
||||
results = [
|
||||
*[_subgroup_result("strong", 10, 0, 0) for _ in range(5)],
|
||||
*[_subgroup_result("weak", 1, 4, 4) for _ in range(5)],
|
||||
]
|
||||
report = subgroup_report(results)
|
||||
region = report["dimensions"]["region"]
|
||||
weak = region["strata"]["weak"]["task_metrics"]["object_detection"]
|
||||
|
||||
assert weak["status"] == "evaluable"
|
||||
assert weak["case_support"] == 5
|
||||
assert weak["micro"]["precision_ci95_wilson"]["status"] == "computed"
|
||||
assert weak["macro"]["f1_case_support"] == 5
|
||||
assert region["worst_stratum_by_task"]["object_detection"]["stratum"] == "weak"
|
||||
|
||||
insufficient = subgroup_report([_subgroup_result("thin", 1, 0, 0)])
|
||||
thin = insufficient["dimensions"]["region"]["strata"]["thin"]
|
||||
assert thin["task_metrics"]["object_detection"]["status"] == "insufficient_support"
|
||||
assert thin["release_gate_status"] == "not_evaluable"
|
||||
assert insufficient["overall_status"] == "not_evaluable"
|
||||
|
||||
|
||||
def test_capability_inventory_is_comprehensive_and_honest() -> None:
|
||||
inventory = task_inventory()
|
||||
assert {item["task"] for item in inventory} == TASKS
|
||||
assert len(inventory) >= 15
|
||||
assert all(item["implementation_paths"] for item in inventory)
|
||||
assert all(item["suitable_metrics"] for item in inventory)
|
||||
assistant = next(
|
||||
item
|
||||
for item in inventory
|
||||
if item["capability_id"] == "geo_assistant_orchestration"
|
||||
)
|
||||
assert assistant["evaluation_status"].startswith("no_independent_accuracy_score")
|
||||
@@ -0,0 +1,222 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
SCRIPTS = ROOT / "scripts"
|
||||
if str(SCRIPTS) not in sys.path:
|
||||
sys.path.insert(0, str(SCRIPTS))
|
||||
|
||||
from generate_accuracy_phase4_splits import ( # noqa: E402
|
||||
LeakageError,
|
||||
assert_training_inputs_safe,
|
||||
build_manifests,
|
||||
generate,
|
||||
)
|
||||
|
||||
|
||||
SOURCE = ROOT / "fixtures/accuracy/p4/split-source-manifest.json"
|
||||
|
||||
|
||||
def load_source() -> dict:
|
||||
return json.loads(SOURCE.read_text(encoding="utf-8"))
|
||||
|
||||
|
||||
def test_normative_roles_hashes_and_source_order_are_enforced() -> None:
|
||||
source = load_source()
|
||||
development, protected, leakage = build_manifests(source)
|
||||
reversed_source = copy.deepcopy(source)
|
||||
reversed_source["samples"].reverse()
|
||||
reversed_development, reversed_protected, reversed_leakage = build_manifests(
|
||||
reversed_source
|
||||
)
|
||||
|
||||
assert leakage["status"] == "pass"
|
||||
assert leakage["finding_count"] == 0
|
||||
assert leakage["split_counts"] == {
|
||||
"background-test": 2,
|
||||
"calibration": 2,
|
||||
"challenge": 4,
|
||||
"test": 7,
|
||||
"train": 3,
|
||||
"val": 3,
|
||||
}
|
||||
assert leakage["crs_validation"] == {
|
||||
"status": "pass",
|
||||
"crs": "EPSG:31370",
|
||||
"distance_units": "m",
|
||||
}
|
||||
assert development["training_access_allowed_by_split"] == {
|
||||
"train": True,
|
||||
"val": False,
|
||||
"calibration": False,
|
||||
}
|
||||
assert protected["labels_available_by_split"]["challenge"] == "sealed_external"
|
||||
assert reversed_development["manifest_sha256"] == development["manifest_sha256"]
|
||||
assert reversed_protected["manifest_sha256"] == protected["manifest_sha256"]
|
||||
assert reversed_leakage == leakage
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("field", "expected_code"),
|
||||
[
|
||||
("group_id", "S-SPATIAL-GROUP"),
|
||||
("source_family", "S-SOURCE-FAMILY"),
|
||||
("temporal_family", "S-TEMPORAL-FAMILY"),
|
||||
("raw_image_sha256", "S-RAW-IMAGE-DUPLICATE"),
|
||||
("processed_image_sha256", "S-PROCESSED-IMAGE-DUPLICATE"),
|
||||
("label_sha256", "S-LABEL-DUPLICATE"),
|
||||
("label_geometry_hash", "S-LABEL-GEOMETRY-DUPLICATE"),
|
||||
("parent_raster_id", "S-PARENT-RASTER"),
|
||||
("acquisition_id", "S-ACQUISITION"),
|
||||
],
|
||||
)
|
||||
def test_cross_split_lineage_and_content_collisions_fail(
|
||||
field: str, expected_code: str
|
||||
) -> None:
|
||||
source = load_source()
|
||||
source["samples"][8][field] = source["samples"][0][field]
|
||||
_development, _protected, leakage = build_manifests(source)
|
||||
|
||||
assert leakage["status"] == "fail"
|
||||
assert expected_code in {item["code"] for item in leakage["findings"]}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("field", "expected_code"),
|
||||
[
|
||||
("perceptual_image_hash", "S-PERCEPTUAL-IMAGE-NEAR-DUPLICATE"),
|
||||
("label_geometry_fingerprint", "S-LABEL-GEOMETRY-NEAR-DUPLICATE"),
|
||||
],
|
||||
)
|
||||
def test_near_duplicate_fingerprints_fail(field: str, expected_code: str) -> None:
|
||||
source = load_source()
|
||||
source["samples"][8][field] = source["samples"][0][field]
|
||||
_development, _protected, leakage = build_manifests(source)
|
||||
|
||||
assert leakage["status"] == "fail"
|
||||
assert expected_code in {item["code"] for item in leakage["findings"]}
|
||||
|
||||
|
||||
def test_object_native_feature_and_spatial_collisions_fail() -> None:
|
||||
source = load_source()
|
||||
source["samples"][8]["object_ids"] = source["samples"][0]["object_ids"]
|
||||
source["samples"][9]["native_feature_ids"] = source["samples"][1][
|
||||
"native_feature_ids"
|
||||
]
|
||||
source["samples"][10]["bbox"] = source["samples"][2]["bbox"]
|
||||
_development, _protected, leakage = build_manifests(source)
|
||||
codes = {item["code"] for item in leakage["findings"]}
|
||||
|
||||
assert {"S-OBJECT-INSTANCE", "S-NATIVE-FEATURE", "S-SPATIAL-OVERLAP"} <= codes
|
||||
|
||||
|
||||
def test_non_metric_crs_and_missing_normative_role_fail_closed() -> None:
|
||||
geographic = load_source()
|
||||
geographic["crs"] = "EPSG:4326"
|
||||
with pytest.raises(LeakageError, match="projected in metres"):
|
||||
build_manifests(geographic)
|
||||
|
||||
missing = load_source()
|
||||
missing["samples"] = [
|
||||
item for item in missing["samples"] if item["split"] != "calibration"
|
||||
]
|
||||
with pytest.raises(LeakageError, match="Required splits are absent"):
|
||||
build_manifests(missing)
|
||||
|
||||
|
||||
def test_training_firewall_only_allows_train_and_binds_protected_lineage() -> None:
|
||||
development, protected, leakage = build_manifests(load_source())
|
||||
assert leakage["status"] == "pass"
|
||||
train = [item for item in development["samples"] if item["split"] == "train"]
|
||||
validation = next(item for item in development["samples"] if item["split"] == "val")
|
||||
protected_item = protected["samples"][0]
|
||||
|
||||
assert_training_inputs_safe([], train, protected)
|
||||
with pytest.raises(LeakageError, match="non_train_role"):
|
||||
assert_training_inputs_safe([], [validation], protected)
|
||||
with pytest.raises(LeakageError, match="protected_identity"):
|
||||
disguised = copy.deepcopy(train[0])
|
||||
disguised["source_family"] = protected_item["source_family"]
|
||||
assert_training_inputs_safe([], [disguised], protected)
|
||||
with pytest.raises(LeakageError, match="protected_path"):
|
||||
assert_training_inputs_safe([Path("vault/protected/test.json")], [], protected)
|
||||
|
||||
|
||||
def test_failed_generation_writes_status_but_no_consumable_manifests(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
source = load_source()
|
||||
source["samples"][8]["group_id"] = source["samples"][0]["group_id"]
|
||||
source_path = tmp_path / "source.json"
|
||||
source_path.write_text(json.dumps(source), encoding="utf-8")
|
||||
output = tmp_path / "out"
|
||||
|
||||
with pytest.raises(LeakageError, match="Leakage gate failed"):
|
||||
generate(source_path, output)
|
||||
|
||||
status = json.loads((output / "generation-status.json").read_text(encoding="utf-8"))
|
||||
assert status["status"] == "fail"
|
||||
assert not (output / "development-split-manifest.json").exists()
|
||||
assert not (output / "protected-split-manifest.json").exists()
|
||||
|
||||
|
||||
def test_deterministic_grouped_assignment_is_stable_and_keeps_relatives_together() -> (
|
||||
None
|
||||
):
|
||||
source = load_source()
|
||||
source["assignment_mode"] = "deterministic_grouped"
|
||||
source["split_assignment"] = {
|
||||
"seed": "fixed-phase4-test-seed",
|
||||
"roles": [
|
||||
"train",
|
||||
"val",
|
||||
"calibration",
|
||||
"test",
|
||||
"background-test",
|
||||
"challenge",
|
||||
],
|
||||
"weights": {
|
||||
"train": 6,
|
||||
"val": 2,
|
||||
"calibration": 1,
|
||||
"test": 2,
|
||||
"background-test": 1,
|
||||
"challenge": 1,
|
||||
},
|
||||
"stratify_by": ["task"],
|
||||
}
|
||||
for item in source["samples"]:
|
||||
item.pop("split")
|
||||
source["samples"][1]["group_id"] = source["samples"][0]["group_id"]
|
||||
|
||||
development, protected, leakage = build_manifests(source)
|
||||
reversed_source = copy.deepcopy(source)
|
||||
reversed_source["samples"].reverse()
|
||||
reversed_development, reversed_protected, reversed_leakage = build_manifests(
|
||||
reversed_source
|
||||
)
|
||||
|
||||
assigned = {
|
||||
item["sample_id"]: item["split"]
|
||||
for item in development["samples"] + protected["samples"]
|
||||
}
|
||||
assert assigned["det-train-a"] == assigned["seg-train-a"]
|
||||
assert set(leakage["split_counts"]) == {
|
||||
"train",
|
||||
"val",
|
||||
"calibration",
|
||||
"test",
|
||||
"background-test",
|
||||
"challenge",
|
||||
}
|
||||
assert leakage["status"] == "pass"
|
||||
assert reversed_development["manifest_sha256"] == development["manifest_sha256"]
|
||||
assert reversed_protected["manifest_sha256"] == protected["manifest_sha256"]
|
||||
assert reversed_leakage == leakage
|
||||
Reference in New Issue
Block a user