diff --git a/backend/tests/test_accuracy_phase4_evaluation.py b/backend/tests/test_accuracy_phase4_evaluation.py index bf8c21bf..fb578ce8 100644 --- a/backend/tests/test_accuracy_phase4_evaluation.py +++ b/backend/tests/test_accuracy_phase4_evaluation.py @@ -35,6 +35,7 @@ from run_accuracy_phase4_benchmark import ( # noqa: E402 active_model_availability_gate, build_release_gate_report, canonical_golden_baseline, + readiness_snapshot, firewall_contract_checks, product_baseline_manifest_gate, product_gate_evidence, @@ -1181,12 +1182,76 @@ def test_one_workflow_is_byte_reproducible_complete_and_fail_closed( ) assert input_manifest["product_baseline"]["validation_status"] == "not_evaluable" assert input_manifest["product_baseline"]["artifacts"] == [] + assert "docs/accuracy-program/status.json" not in { + item["path"] for item in input_manifest["inputs"] + } + assert input_manifest["readiness_snapshot"]["source_paths"][ + "accuracy_status_projection" + ]["selected_json_pointers"] == ["/runtime/active_model"] assert benchmark_manifest["product_baseline"] == input_manifest["product_baseline"] assert ( benchmark_manifest["product_gate_evidence_sha256"] == first["product_gate_evidence_sha256"] ) - assert first["evidence_run_id"].startswith("p4-2.0.0-") + assert first["evidence_run_id"].startswith("p4-2.0.1-") + + +def test_readiness_snapshot_ignores_phase4_bookkeeping_but_binds_active_model( + tmp_path: Path, +) -> None: + status_path = tmp_path / "docs/accuracy-program/status.json" + scan_path = tmp_path / "artifacts/evidence/accuracy/P3/full-scan-manifest.json" + leakage_path = tmp_path / "artifacts/evidence/accuracy/P3/leakage-report.json" + status_path.parent.mkdir(parents=True) + scan_path.parent.mkdir(parents=True) + status = { + "generated_at": "2026-08-02T00:00:00+02:00", + "documents": ["old.md"], + "phase5": {"status": "not_ready"}, + "verification": {"phase4_evaluation": {"status": "pending"}}, + "phase4": {"status": "ready"}, + "runtime": { + "active_model": { + "model_id": "model-a", + "path": "/models/a.pt", + "sha256": "a" * 64, + } + }, + } + status_path.write_text(json.dumps(status), encoding="utf-8") + scan_path.write_text( + json.dumps({"scan_id": "scan-a", "content_hash": "b" * 64}), + encoding="utf-8", + ) + leakage_path.write_text(json.dumps({"status": "attention"}), encoding="utf-8") + + original = readiness_snapshot(tmp_path) + status["phase4"] = {"status": "in_progress", "evidence_run_id": "run-a"} + status["generated_at"] = "2026-08-02T05:00:00+02:00" + status["documents"] = ["old.md", "new.md"] + status["phase5"] = {"status": "blocked"} + status["verification"] = {"phase4_evaluation": {"status": "local_pass"}} + status_path.write_text(json.dumps(status), encoding="utf-8") + bookkeeping_update = readiness_snapshot(tmp_path) + + assert bookkeeping_update == original + assert "accuracy_status" not in original["source_paths"] + assert original["source_paths"]["accuracy_status_projection"][ + "selected_json_pointers" + ] == ["/runtime/active_model"] + + status["runtime"]["active_model"]["sha256"] = "c" * 64 + status_path.write_text(json.dumps(status), encoding="utf-8") + assert readiness_snapshot(tmp_path) != original + + assert original["source_paths"]["accuracy_status_projection"][ + "sha256" + ] == canonical_hash( + { + "schema_version": 1, + "runtime": {"active_model": original["active_model"]}, + } + ) def test_immutable_workflow_refuses_to_replace_changed_evidence(tmp_path: Path) -> None: diff --git a/scripts/run_accuracy_phase4_benchmark.py b/scripts/run_accuracy_phase4_benchmark.py index 43004850..7327a967 100644 --- a/scripts/run_accuracy_phase4_benchmark.py +++ b/scripts/run_accuracy_phase4_benchmark.py @@ -42,7 +42,7 @@ from generate_accuracy_phase4_splits import ( # noqa: E402 ) from run_golden_qa_benchmark import run_benchmark # noqa: E402 -WORKFLOW_VERSION = "2.0.0" +WORKFLOW_VERSION = "2.0.1" BENCHMARK_ID = "geointel-p4-reference-harness-v2" GATE_STATES = {"pass", "fail", "not_evaluable"} PRODUCT_BASELINE_SCHEMA_VERSION = 2 @@ -2154,13 +2154,18 @@ def readiness_snapshot(repo_root: Path) -> dict[str, Any]: status = json.loads(status_path.read_text(encoding="utf-8")) p3 = json.loads(p3_path.read_text(encoding="utf-8")) leakage = json.loads(leakage_path.read_text(encoding="utf-8")) - ml_data = status.get("ml_data") or {} + status_projection = { + "schema_version": 1, + "runtime": {"active_model": (status.get("runtime") or {}).get("active_model")}, + } return { "schema_version": 1, "source_paths": { - "accuracy_status": repository_file( - repo_root, "docs/accuracy-program/status.json" - ), + "accuracy_status_projection": { + "path": "docs/accuracy-program/status.json", + "selected_json_pointers": ["/runtime/active_model"], + "sha256": canonical_hash(status_projection), + }, "phase3_full_scan": repository_file( repo_root, "artifacts/evidence/accuracy/P3/full-scan-manifest.json" ), @@ -2168,9 +2173,7 @@ def readiness_snapshot(repo_root: Path) -> dict[str, Any]: repo_root, "artifacts/evidence/accuracy/P3/leakage-report.json" ), }, - "active_model": (status.get("runtime") or {}).get("active_model"), - "v56_review_and_split": (ml_data.get("v56") or {}), - "protected_test_isolation": ml_data.get("protected_test_isolation"), + "active_model": status_projection["runtime"]["active_model"], "phase3_scan": { "scan_id": p3.get("scan_id"), "content_hash": p3.get("content_hash"), @@ -2607,7 +2610,6 @@ def build_input_manifest( "fixtures/accuracy/p4/split-source-manifest.json", "fixtures/accuracy/p4/protected-baseline-cases.json", "fixtures/golden/golden_qa_benchmarks.json", - "docs/accuracy-program/status.json", "docs/accuracy-program/05-metric-framework.md", "docs/accuracy-program/07-source-authority-matrix.md", "artifacts/evidence/accuracy/P3/full-scan-manifest.json",