feat(accuracy): build hardened phase 4 evaluation harness

2026-08-02 02:43:41 +02:00
parent 7b92e29e49
commit bd2fd9780f
13 changed files with 5468 additions and 7 deletions
@@ -0,0 +1,240 @@
+from __future__ import annotations
+
+import copy
+import hashlib
+import json
+import sys
+from pathlib import Path
+
+import pytest
+
+
+ROOT = Path(__file__).resolve().parents[2]
+SCRIPTS = ROOT / "scripts"
+if str(SCRIPTS) not in sys.path:
+    sys.path.insert(0, str(SCRIPTS))
+
+from accuracy_phase4_evaluator import evaluate_cases  # noqa: E402
+from generate_accuracy_phase4_splits import (  # noqa: E402
+    LeakageError,
+    assert_training_inputs_safe,
+    build_manifests,
+)
+from run_accuracy_phase4_benchmark import (  # noqa: E402
+    EvidenceConflictError,
+    build_release_gate_report,
+    canonical_golden_baseline,
+    firewall_contract_checks,
+    run_workflow,
+)
+
+
+SOURCE = ROOT / "fixtures/accuracy/p4/split-source-manifest.json"
+CASES = ROOT / "fixtures/accuracy/p4/protected-baseline-cases.json"
+
+
+def load_source() -> dict:
+    return json.loads(SOURCE.read_text(encoding="utf-8"))
+
+
+def evaluation_inputs() -> tuple[dict, dict, dict, dict]:
+    source = load_source()
+    development, protected, leakage = build_manifests(source)
+    split_result = {
+        "development": development,
+        "protected": protected,
+        "leakage": leakage,
+    }
+    allowed = {
+        item["sample_id"]
+        for item in protected["samples"]
+        if item["split"] in {"test", "background-test"}
+    }
+    evaluation = evaluate_cases(CASES, allowed)
+    portfolio = json.loads(CASES.read_text(encoding="utf-8"))
+    firewall = firewall_contract_checks(split_result, CASES)
+    return split_result, evaluation, portfolio, firewall
+
+
+def test_split_fixture_is_order_independent_and_has_all_roles() -> None:
+    source = load_source()
+    development, protected, leakage = build_manifests(source)
+    reversed_source = copy.deepcopy(source)
+    reversed_source["samples"].reverse()
+    reversed_development, reversed_protected, reversed_leakage = build_manifests(
+        reversed_source
+    )
+
+    assert leakage["status"] == "pass"
+    assert leakage["finding_count"] == 0
+    assert leakage["split_counts"] == {
+        "background-test": 2,
+        "calibration": 2,
+        "challenge": 4,
+        "test": 7,
+        "train": 3,
+        "val": 3,
+    }
+    assert reversed_development["manifest_sha256"] == development["manifest_sha256"]
+    assert reversed_protected["manifest_sha256"] == protected["manifest_sha256"]
+    assert reversed_leakage == leakage
+
+
+def test_training_firewall_rejects_non_train_and_protected_lineage() -> None:
+    development, protected, leakage = build_manifests(load_source())
+    assert leakage["status"] == "pass"
+    train = [item for item in development["samples"] if item["split"] == "train"]
+    validation = next(item for item in development["samples"] if item["split"] == "val")
+
+    assert_training_inputs_safe([], train, protected)
+    with pytest.raises(LeakageError, match="non_train_role"):
+        assert_training_inputs_safe([], [validation], protected)
+    with pytest.raises(LeakageError, match="protected_identity"):
+        disguised = dict(train[0])
+        disguised["source_family"] = protected["samples"][0]["source_family"]
+        assert_training_inputs_safe([], [disguised], protected)
+    with pytest.raises(LeakageError, match="protected_path"):
+        assert_training_inputs_safe([CASES], [], protected)
+
+
+def test_task_evaluator_retains_exact_raw_inputs_metrics_and_failures() -> None:
+    _split_result, report, _portfolio, _firewall = evaluation_inputs()
+
+    assert report["task_count"] == 7
+    assert report["case_count"] == 9
+    assert len(report["task_inventory"]) >= 15
+    assert len(report["failures"]) == 11
+    assert report["subgroups"]["overall_status"] == "not_evaluable"
+    assert all(
+        {
+            "references",
+            "predictions_pre_filter",
+            "predictions_post_filter",
+            "config",
+            "input_lineage",
+            "portfolio_lineage",
+        }
+        <= set(item["raw"])
+        for item in report["results"]
+    )
+    detection = next(
+        item
+        for item in report["results"]
+        if item["sample_id"] == "det-test-flanders-urban"
+    )
+    assert len(detection["raw"]["predictions_pre_filter"]) == 4
+    assert len(detection["raw"]["predictions_post_filter"]) == 3
+    assert detection["metrics"]["true_positive"] == 2
+    assert detection["metrics"]["false_positive"] == 1
+    assert detection["metrics"]["ap50"] is not None
+    empty = next(
+        item
+        for item in report["results"]
+        if item["sample_id"] == "background-test-pure-empty"
+    )
+    assert empty["metrics"]["precision"] is None
+    assert empty["metrics"]["recall"] is None
+    assert empty["metrics"]["f1"] is None
+
+
+def test_product_prerequisites_cannot_pass_without_executed_baseline() -> None:
+    split_result, evaluation, portfolio, firewall = evaluation_inputs()
+    product_gates = {
+        "active_model_available_and_hash_verified": {"status": "pass"},
+        "authoritative_reference_portfolio_available": {"status": "pass"},
+        "human_review_complete": {"status": "pass"},
+        "split_independence": {"status": "pass"},
+        "phase3_leakage_resolved": {"status": "pass"},
+        "protected_storage_isolation": {"status": "pass"},
+        "executed_product_incumbent_baseline": {
+            "status": "not_evaluable",
+            "reason": "no raw active-model inference",
+        },
+        "representative_product_subgroup_support": {"status": "pass"},
+    }
+    report = build_release_gate_report(
+        split_result,
+        evaluation,
+        portfolio,
+        canonical_golden_baseline(),
+        firewall,
+        product_gates,
+    )
+
+    assert report["local_harness_status"] == "pass"
+    assert report["product_benchmark_status"] == "not_evaluable"
+    assert report["status"] == "not_evaluable"
+    assert report["phase_decision"] == "blocked"
+
+
+def test_one_workflow_is_byte_reproducible_complete_and_fail_closed(
+    tmp_path: Path,
+) -> None:
+    output = tmp_path / "p4"
+    first = run_workflow(ROOT, output)
+    first_bytes = {path.name: path.read_bytes() for path in output.glob("*.json")}
+    second = run_workflow(ROOT, output)
+    second_bytes = {path.name: path.read_bytes() for path in output.glob("*.json")}
+
+    assert first == second
+    assert first_bytes == second_bytes
+    assert first["local_harness_status"] == "pass"
+    assert first["product_benchmark_status"] == "not_evaluable"
+    assert first["status"] == "not_evaluable"
+    assert first["phase4_done"] is False
+    assert first["phase5_ready"] is False
+    required = {
+        "acceptance-gates.json",
+        "aoi-metrics.json",
+        "baseline-raw-predictions.json",
+        "benchmark-manifest.json",
+        "calibration-metrics.json",
+        "candidate-vs-incumbent.json",
+        "development-split-manifest.json",
+        "error-taxonomy.json",
+        "evaluation-contract.json",
+        "failure-gallery.json",
+        "generation-status.json",
+        "human-review-summary.json",
+        "input-manifest.json",
+        "latency-and-reliability.json",
+        "leakage-gate-report.json",
+        "metric-report.json",
+        "object-metrics.json",
+        "protected-split-manifest.json",
+        "reference-implementation-baseline.json",
+        "release-gate-report.json",
+        "split-and-leakage-audit.json",
+        "stratified-metrics.json",
+        "tile-metrics.json",
+        "workflow-summary.json",
+        "evidence-manifest.json",
+    }
+    assert required == set(first_bytes)
+    manifest = json.loads(
+        (output / "evidence-manifest.json").read_text(encoding="utf-8")
+    )
+    assert manifest["artifact_count"] == len(required) - 1
+    for item in manifest["artifacts"]:
+        path = output / item["path"]
+        assert path.stat().st_size == item["size_bytes"]
+        assert hashlib.sha256(path.read_bytes()).hexdigest() == item["sha256"]
+    gates = json.loads(
+        (output / "release-gate-report.json").read_text(encoding="utf-8")
+    )
+    assert gates["promotion_allowed"] is False
+    assert all(item["status"] == "pass" for item in gates["local_gates"].values())
+    assert {item["status"] for item in gates["product_gates"].values()} <= {
+        "pass",
+        "fail",
+        "not_evaluable",
+    }
+
+
+def test_immutable_workflow_refuses_to_replace_changed_evidence(tmp_path: Path) -> None:
+    output = tmp_path / "p4"
+    run_workflow(ROOT, output)
+    (output / "workflow-summary.json").write_text("{}\n", encoding="utf-8")
+
+    with pytest.raises(EvidenceConflictError, match="Refusing to overwrite"):
+        run_workflow(ROOT, output)