241 lines
8.4 KiB
Python
241 lines
8.4 KiB
Python
from __future__ import annotations
|
|
|
|
import copy
|
|
import hashlib
|
|
import json
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
|
|
ROOT = Path(__file__).resolve().parents[2]
|
|
SCRIPTS = ROOT / "scripts"
|
|
if str(SCRIPTS) not in sys.path:
|
|
sys.path.insert(0, str(SCRIPTS))
|
|
|
|
from accuracy_phase4_evaluator import evaluate_cases # noqa: E402
|
|
from generate_accuracy_phase4_splits import ( # noqa: E402
|
|
LeakageError,
|
|
assert_training_inputs_safe,
|
|
build_manifests,
|
|
)
|
|
from run_accuracy_phase4_benchmark import ( # noqa: E402
|
|
EvidenceConflictError,
|
|
build_release_gate_report,
|
|
canonical_golden_baseline,
|
|
firewall_contract_checks,
|
|
run_workflow,
|
|
)
|
|
|
|
|
|
SOURCE = ROOT / "fixtures/accuracy/p4/split-source-manifest.json"
|
|
CASES = ROOT / "fixtures/accuracy/p4/protected-baseline-cases.json"
|
|
|
|
|
|
def load_source() -> dict:
|
|
return json.loads(SOURCE.read_text(encoding="utf-8"))
|
|
|
|
|
|
def evaluation_inputs() -> tuple[dict, dict, dict, dict]:
|
|
source = load_source()
|
|
development, protected, leakage = build_manifests(source)
|
|
split_result = {
|
|
"development": development,
|
|
"protected": protected,
|
|
"leakage": leakage,
|
|
}
|
|
allowed = {
|
|
item["sample_id"]
|
|
for item in protected["samples"]
|
|
if item["split"] in {"test", "background-test"}
|
|
}
|
|
evaluation = evaluate_cases(CASES, allowed)
|
|
portfolio = json.loads(CASES.read_text(encoding="utf-8"))
|
|
firewall = firewall_contract_checks(split_result, CASES)
|
|
return split_result, evaluation, portfolio, firewall
|
|
|
|
|
|
def test_split_fixture_is_order_independent_and_has_all_roles() -> None:
|
|
source = load_source()
|
|
development, protected, leakage = build_manifests(source)
|
|
reversed_source = copy.deepcopy(source)
|
|
reversed_source["samples"].reverse()
|
|
reversed_development, reversed_protected, reversed_leakage = build_manifests(
|
|
reversed_source
|
|
)
|
|
|
|
assert leakage["status"] == "pass"
|
|
assert leakage["finding_count"] == 0
|
|
assert leakage["split_counts"] == {
|
|
"background-test": 2,
|
|
"calibration": 2,
|
|
"challenge": 4,
|
|
"test": 7,
|
|
"train": 3,
|
|
"val": 3,
|
|
}
|
|
assert reversed_development["manifest_sha256"] == development["manifest_sha256"]
|
|
assert reversed_protected["manifest_sha256"] == protected["manifest_sha256"]
|
|
assert reversed_leakage == leakage
|
|
|
|
|
|
def test_training_firewall_rejects_non_train_and_protected_lineage() -> None:
|
|
development, protected, leakage = build_manifests(load_source())
|
|
assert leakage["status"] == "pass"
|
|
train = [item for item in development["samples"] if item["split"] == "train"]
|
|
validation = next(item for item in development["samples"] if item["split"] == "val")
|
|
|
|
assert_training_inputs_safe([], train, protected)
|
|
with pytest.raises(LeakageError, match="non_train_role"):
|
|
assert_training_inputs_safe([], [validation], protected)
|
|
with pytest.raises(LeakageError, match="protected_identity"):
|
|
disguised = dict(train[0])
|
|
disguised["source_family"] = protected["samples"][0]["source_family"]
|
|
assert_training_inputs_safe([], [disguised], protected)
|
|
with pytest.raises(LeakageError, match="protected_path"):
|
|
assert_training_inputs_safe([CASES], [], protected)
|
|
|
|
|
|
def test_task_evaluator_retains_exact_raw_inputs_metrics_and_failures() -> None:
|
|
_split_result, report, _portfolio, _firewall = evaluation_inputs()
|
|
|
|
assert report["task_count"] == 7
|
|
assert report["case_count"] == 9
|
|
assert len(report["task_inventory"]) >= 15
|
|
assert len(report["failures"]) == 11
|
|
assert report["subgroups"]["overall_status"] == "not_evaluable"
|
|
assert all(
|
|
{
|
|
"references",
|
|
"predictions_pre_filter",
|
|
"predictions_post_filter",
|
|
"config",
|
|
"input_lineage",
|
|
"portfolio_lineage",
|
|
}
|
|
<= set(item["raw"])
|
|
for item in report["results"]
|
|
)
|
|
detection = next(
|
|
item
|
|
for item in report["results"]
|
|
if item["sample_id"] == "det-test-flanders-urban"
|
|
)
|
|
assert len(detection["raw"]["predictions_pre_filter"]) == 4
|
|
assert len(detection["raw"]["predictions_post_filter"]) == 3
|
|
assert detection["metrics"]["true_positive"] == 2
|
|
assert detection["metrics"]["false_positive"] == 1
|
|
assert detection["metrics"]["ap50"] is not None
|
|
empty = next(
|
|
item
|
|
for item in report["results"]
|
|
if item["sample_id"] == "background-test-pure-empty"
|
|
)
|
|
assert empty["metrics"]["precision"] is None
|
|
assert empty["metrics"]["recall"] is None
|
|
assert empty["metrics"]["f1"] is None
|
|
|
|
|
|
def test_product_prerequisites_cannot_pass_without_executed_baseline() -> None:
|
|
split_result, evaluation, portfolio, firewall = evaluation_inputs()
|
|
product_gates = {
|
|
"active_model_available_and_hash_verified": {"status": "pass"},
|
|
"authoritative_reference_portfolio_available": {"status": "pass"},
|
|
"human_review_complete": {"status": "pass"},
|
|
"split_independence": {"status": "pass"},
|
|
"phase3_leakage_resolved": {"status": "pass"},
|
|
"protected_storage_isolation": {"status": "pass"},
|
|
"executed_product_incumbent_baseline": {
|
|
"status": "not_evaluable",
|
|
"reason": "no raw active-model inference",
|
|
},
|
|
"representative_product_subgroup_support": {"status": "pass"},
|
|
}
|
|
report = build_release_gate_report(
|
|
split_result,
|
|
evaluation,
|
|
portfolio,
|
|
canonical_golden_baseline(),
|
|
firewall,
|
|
product_gates,
|
|
)
|
|
|
|
assert report["local_harness_status"] == "pass"
|
|
assert report["product_benchmark_status"] == "not_evaluable"
|
|
assert report["status"] == "not_evaluable"
|
|
assert report["phase_decision"] == "blocked"
|
|
|
|
|
|
def test_one_workflow_is_byte_reproducible_complete_and_fail_closed(
|
|
tmp_path: Path,
|
|
) -> None:
|
|
output = tmp_path / "p4"
|
|
first = run_workflow(ROOT, output)
|
|
first_bytes = {path.name: path.read_bytes() for path in output.glob("*.json")}
|
|
second = run_workflow(ROOT, output)
|
|
second_bytes = {path.name: path.read_bytes() for path in output.glob("*.json")}
|
|
|
|
assert first == second
|
|
assert first_bytes == second_bytes
|
|
assert first["local_harness_status"] == "pass"
|
|
assert first["product_benchmark_status"] == "not_evaluable"
|
|
assert first["status"] == "not_evaluable"
|
|
assert first["phase4_done"] is False
|
|
assert first["phase5_ready"] is False
|
|
required = {
|
|
"acceptance-gates.json",
|
|
"aoi-metrics.json",
|
|
"baseline-raw-predictions.json",
|
|
"benchmark-manifest.json",
|
|
"calibration-metrics.json",
|
|
"candidate-vs-incumbent.json",
|
|
"development-split-manifest.json",
|
|
"error-taxonomy.json",
|
|
"evaluation-contract.json",
|
|
"failure-gallery.json",
|
|
"generation-status.json",
|
|
"human-review-summary.json",
|
|
"input-manifest.json",
|
|
"latency-and-reliability.json",
|
|
"leakage-gate-report.json",
|
|
"metric-report.json",
|
|
"object-metrics.json",
|
|
"protected-split-manifest.json",
|
|
"reference-implementation-baseline.json",
|
|
"release-gate-report.json",
|
|
"split-and-leakage-audit.json",
|
|
"stratified-metrics.json",
|
|
"tile-metrics.json",
|
|
"workflow-summary.json",
|
|
"evidence-manifest.json",
|
|
}
|
|
assert required == set(first_bytes)
|
|
manifest = json.loads(
|
|
(output / "evidence-manifest.json").read_text(encoding="utf-8")
|
|
)
|
|
assert manifest["artifact_count"] == len(required) - 1
|
|
for item in manifest["artifacts"]:
|
|
path = output / item["path"]
|
|
assert path.stat().st_size == item["size_bytes"]
|
|
assert hashlib.sha256(path.read_bytes()).hexdigest() == item["sha256"]
|
|
gates = json.loads(
|
|
(output / "release-gate-report.json").read_text(encoding="utf-8")
|
|
)
|
|
assert gates["promotion_allowed"] is False
|
|
assert all(item["status"] == "pass" for item in gates["local_gates"].values())
|
|
assert {item["status"] for item in gates["product_gates"].values()} <= {
|
|
"pass",
|
|
"fail",
|
|
"not_evaluable",
|
|
}
|
|
|
|
|
|
def test_immutable_workflow_refuses_to_replace_changed_evidence(tmp_path: Path) -> None:
|
|
output = tmp_path / "p4"
|
|
run_workflow(ROOT, output)
|
|
(output / "workflow-summary.json").write_text("{}\n", encoding="utf-8")
|
|
|
|
with pytest.raises(EvidenceConflictError, match="Refusing to overwrite"):
|
|
run_workflow(ROOT, output)
|