from __future__ import annotations import copy import hashlib import json import sys from pathlib import Path from typing import Any import pytest ROOT = Path(__file__).resolve().parents[2] SCRIPTS = ROOT / "scripts" if str(SCRIPTS) not in sys.path: sys.path.insert(0, str(SCRIPTS)) from accuracy_phase4_evaluator import ( # noqa: E402 TASKS, canonical_hash, evaluate_cases, task_inventory, ) from generate_accuracy_phase4_splits import ( # noqa: E402 LeakageError, assert_training_inputs_safe, build_manifests, ) from run_accuracy_phase4_benchmark import ( # noqa: E402 EvidenceConflictError, PRODUCT_GATE_NAMES, REQUIRED_AUTHORITY_REQUIREMENTS, REQUIRED_SUBGROUP_DIMENSION_FIELDS, SUBGROUP_RELEASE_POLICY, active_model_availability_gate, build_release_gate_report, canonical_golden_baseline, readiness_snapshot, firewall_contract_checks, product_baseline_manifest_gate, product_gate_evidence, run_workflow, ) SOURCE = ROOT / "fixtures/accuracy/p4/split-source-manifest.json" CASES = ROOT / "fixtures/accuracy/p4/protected-baseline-cases.json" def _fixture_hash(value: str) -> str: return hashlib.sha256(value.encode("utf-8")).hexdigest() def _write_governed_json( repo_root: Path, path: Path, payload: dict[str, Any], ) -> dict[str, Any]: path.parent.mkdir(parents=True, exist_ok=True) content = ( json.dumps(payload, ensure_ascii=False, indent=2, sort_keys=True) + "\n" ).encode("utf-8") path.write_bytes(content) return { "path": path.relative_to(repo_root).as_posix(), "sha256": hashlib.sha256(content).hexdigest(), "size_bytes": len(content), } def _build_governed_product_fixture(tmp_path: Path) -> dict[str, Any]: """Build structural governance evidence; this fixture makes no accuracy claim.""" repo_root = tmp_path / "repo" evaluator_path = repo_root / "scripts/accuracy_phase4_evaluator.py" evaluator_path.parent.mkdir(parents=True, exist_ok=True) evaluator_path.write_text("# governed evaluator fixture\n", encoding="utf-8") evaluator_hash = hashlib.sha256(evaluator_path.read_bytes()).hexdigest() model_path = repo_root / "models/active.pt" model_path.parent.mkdir(parents=True, exist_ok=True) model_path.write_bytes(b"governed-model-fixture") active_model = { "model_id": "fixture-model", "model_version": "1.0.0", "path": str(model_path), "sha256": hashlib.sha256(model_path.read_bytes()).hexdigest(), "size_bytes": model_path.stat().st_size, } baseline_id = "governed-product-fixture" evidence_root = repo_root / "artifacts/evidence/accuracy/P4/governed-fixture" raw_path = evidence_root / "raw-predictions.json" development_split_hash = _fixture_hash("governed-development-split-v1") source_portfolio = json.loads(CASES.read_text(encoding="utf-8")) templates: dict[str, dict[str, Any]] = {} for case in source_portfolio["cases"]: templates.setdefault(case["task"], case) assert set(templates) == TASKS profiles = ( { "region": "flanders", "municipality": "Mol", "urbanity": "urban", "object_size": "small", "source": "governed-grb-orthophoto", "sensor": "aerial-rgb", "resolution_m": 0.25, "season": "summer", "date": "2025-06-15", "vegetation": "low", "occlusion": "none", "difficulty": "normal", "context": "dense_urban", }, { "region": "wallonia", "municipality": "Namur", "urbanity": "rural", "object_size": "large", "source": "governed-picc-orthophoto", "sensor": "multispectral-rgb", "resolution_m": 1.0, "season": "winter", "date": "2025-01-15", "vegetation": "high", "occlusion": "partial", "difficulty": "hard", "context": "rural_occluded", }, ) authority_scopes = [ { "task": requirement["task"], "zone": requirement["zone"], "authority": requirement["primary"], } for requirement in REQUIRED_AUTHORITY_REQUIREMENTS ] cases: list[dict[str, Any]] = [] protected_samples: list[dict[str, Any]] = [] parameters_by_task = { task: copy.deepcopy(templates[task]["config"]) for task in sorted(TASKS) } for task in sorted(TASKS): for profile_index, profile in enumerate(profiles): stratum = "a" if profile_index == 0 else "b" for replicate in range(5): sample_id = f"governed-{task}-{stratum}-{replicate}" case = copy.deepcopy(templates[task]) case["sample_id"] = sample_id case["split"] = ( "background-test" if task == "object_detection" and profile_index == 1 else "test" ) case["metadata"].update(profile) case["metadata"].update( { "tile_edge": profile_index == 1, "label_review_state": "human_reviewed_fixture", "ood": False, } ) case["config"] = copy.deepcopy(parameters_by_task[task]) case["lineage"] = { "reference": { "source_id": f"governed:{sample_id}:reference", "source_version": "fixture-v1", "derivation": "structural_contract_fixture_reference", }, "prediction": { "source_id": f"governed:{sample_id}:prediction", "source_version": "fixture-v1", "derivation": "structural_contract_fixture_prediction", }, } cases.append(case) reference_payload = ( case["expected_anomalies"] if task == "geospatial_data_validation" else case["references"] ) subgroups = { dimension: case["metadata"][metadata_field] for dimension, metadata_field in ( ("region", "region"), ("municipality", "municipality"), ("urbanity", "urbanity"), ("object_size", "object_size"), ("source", "source"), ("sensor", "sensor"), ("resolution", "resolution_m"), ("season", "season"), ("date", "date"), ("vegetation", "vegetation"), ("occlusion", "occlusion"), ("difficulty", "difficulty"), ("context", "context"), ) } protected_samples.append( { "sample_id": sample_id, "split": case["split"], "task": task, "zone": str(profile["region"]), "aoi_id": f"independent-aoi-{task}-{stratum}-{replicate}", "content_sha256": _fixture_hash(f"content:{sample_id}"), "label_sha256": canonical_hash(reference_payload), "case_input_sha256": canonical_hash(case), "labels_access_policy": "evaluation_only", "subgroups": subgroups, "authority_scopes": copy.deepcopy(authority_scopes), } ) challenge_id = "governed-challenge-sealed" protected_samples.append( { "sample_id": challenge_id, "split": "challenge", "task": "object_detection", "zone": "flanders", "aoi_id": "independent-aoi-challenge-sealed", "content_sha256": _fixture_hash(f"content:{challenge_id}"), "labels_sealed": True, "subgroups": { dimension: profiles[0][metadata_field] for dimension, metadata_field in ( ("region", "region"), ("municipality", "municipality"), ("urbanity", "urbanity"), ("object_size", "object_size"), ("source", "source"), ("sensor", "sensor"), ("resolution", "resolution_m"), ("season", "season"), ("date", "date"), ("vegetation", "vegetation"), ("occlusion", "occlusion"), ("difficulty", "difficulty"), ("context", "context"), ) }, "authority_scopes": copy.deepcopy(authority_scopes), } ) evaluation_ids = sorted(case["sample_id"] for case in cases) all_ids = sorted(sample["sample_id"] for sample in protected_samples) task_sample_ids = { task: sorted(case["sample_id"] for case in cases if case["task"] == task) for task in sorted(TASKS) } split_counts = { split: sum(sample["split"] == split for sample in protected_samples) for split in sorted({sample["split"] for sample in protected_samples}) } protected_split = { "schema_version": 3, "artifact_role": "protected_evaluation_split", "protected_policy": { "immutable": True, "training_allowed": False, "threshold_selection_allowed": False, "model_selection_allowed": False, "iterative_error_correction_allowed": False, "challenge_labels_accessible": False, }, "evaluator_task_inventory_sha256": canonical_hash(task_inventory()), "samples": protected_samples, "split_counts": split_counts, "sample_ids_sha256": canonical_hash(all_ids), "evaluation_sample_ids_sha256": canonical_hash(evaluation_ids), "evaluated_task_families": sorted(TASKS), "task_sample_ids": task_sample_ids, "task_sample_ids_canonical_json_sha256": canonical_hash(task_sample_ids), } protected_descriptor = _write_governed_json( repo_root, evidence_root / "protected-split.json", protected_split, ) sample_references = [ { "sample_id": case["sample_id"], "task": case["task"], "reference_payload_sha256": canonical_hash( case["expected_anomalies"] if case["task"] == "geospatial_data_validation" else case["references"] ), "reference_lineage_sha256": canonical_hash(case["lineage"]["reference"]), } for case in sorted(cases, key=lambda item: item["sample_id"]) ] authority_entries = [ { "task": requirement["task"], "zone": requirement["zone"], "authority": requirement["primary"], "source_classification": "authoritative", "source_snapshot_id": f"snapshot-{requirement['primary']}-2026", "source_snapshot_sha256": _fixture_hash( f"snapshot:{requirement['primary']}" ), "sample_ids": evaluation_ids, } for requirement in REQUIRED_AUTHORITY_REQUIREMENTS ] authority_portfolio = { "schema_version": 2, "artifact_role": "authoritative_reference_portfolio", "portfolio_id": "governed-authority-fixture", "protected_split_sha256": protected_descriptor["sha256"], "entries": authority_entries, "entries_canonical_json_sha256": canonical_hash(authority_entries), "sample_references": sample_references, "sample_references_canonical_json_sha256": canonical_hash(sample_references), } authority_descriptor = _write_governed_json( repo_root, evidence_root / "authoritative-reference.json", authority_portfolio, ) portfolio_lineage = { "origin": "governed_product_inference", "source_path": raw_path.relative_to(repo_root).as_posix(), "version": baseline_id, "active_model_sha256": active_model["sha256"], "configuration_sha256": "0" * 64, "protected_split_sha256": protected_descriptor["sha256"], "authoritative_reference_sha256": authority_descriptor["sha256"], "inference_evidence_sha256": "0" * 64, } raw_portfolio = { "schema_version": 2, "portfolio_kind": "governed_product_baseline", "portfolio_id": baseline_id, "split_roles": ["test", "background-test"], "selection_policy": "frozen_validation_calibration_only_no_protected_selection", "protected_policy": { "operating_point_selection_allowed": False, "diagnostic_curves_select_operating_point": False, "test_feedback_allowed": False, "threshold_selection_source": "pre_registered_configuration_only", }, "portfolio_lineage": portfolio_lineage, "claim_boundary": ( "Governed product baseline structural fixture validates evidence " "contracts only and makes no production accuracy claim." ), "cases": cases, } preliminary_path = repo_root / "preliminary-product-portfolio.json" _write_governed_json(repo_root, preliminary_path, raw_portfolio) preliminary_evaluation = evaluate_cases(preliminary_path, set(evaluation_ids)) preliminary_path.unlink() subgroup_targets = { task: { "metric": metrics["primary_metric"]["name"], "direction": metrics["primary_metric"]["direction"], "threshold": metrics["primary_metric"]["value"], } for task, metrics in preliminary_evaluation["portfolio_metrics"].items() } configuration = { "schema_version": 2, "artifact_role": "frozen_inference_configuration", "active_model_sha256": active_model["sha256"], "development_split_manifest_sha256": development_split_hash, "frozen_before_protected_access": True, "frozen_at": "2026-08-02T09:00:00+00:00", "protected_data_used": False, "threshold_selection_source": "validation_and_calibration", "parameters_by_task": parameters_by_task, "subgroup_release_policy": SUBGROUP_RELEASE_POLICY, "subgroup_release_targets": subgroup_targets, "subgroup_release_targets_sha256": canonical_hash(subgroup_targets), } configuration_descriptor = _write_governed_json( repo_root, evidence_root / "configuration.json", configuration, ) execution_id = "cuda-execution-fixture-001" runtime_observation = { "status": "pass", "device": "cuda:0", "device_name": "NVIDIA governed fixture", "gpu_uuid": "GPU-governed-fixture", "driver_version": "570.00", "cuda_runtime_version": "12.8", "torch_version": "2.7.0", "cuda_device_count": 1, "kernel_execution_confirmed": True, } inference_evidence = { "schema_version": 2, "artifact_role": "governed_cuda_inference_execution", "execution_id": execution_id, "active_model_sha256": active_model["sha256"], "configuration_sha256": configuration_descriptor["sha256"], "evaluator_sha256": evaluator_hash, "protected_split_sha256": protected_descriptor["sha256"], "executed": True, "exit_code": 0, "test_used_for_selection": False, "device_type": "cuda", "device": "cuda:0", "torch_cuda_is_available": True, "cuda_device_count": 1, "kernel_execution_confirmed": True, "batch_failure_count": 0, "torch_version": "2.7.0", "cuda_runtime_version": "12.8", "driver_version": "570.00", "started_at": "2026-08-02T10:00:00+00:00", "finished_at": "2026-08-02T10:05:00+00:00", "nvidia_smi": { "gpu_uuid": "GPU-governed-fixture", "device_name": "NVIDIA governed fixture", "driver_version": "570.00", "cuda_version": "12.8", "query_output_sha256": _fixture_hash("nvidia-smi-output"), }, "processed_sample_ids": evaluation_ids, "processed_sample_ids_sha256": canonical_hash(evaluation_ids), "successful_sample_count": len(evaluation_ids), } inference_descriptor = _write_governed_json( repo_root, evidence_root / "inference-evidence.json", inference_evidence, ) raw_portfolio["portfolio_lineage"]["configuration_sha256"] = ( configuration_descriptor["sha256"] ) raw_portfolio["portfolio_lineage"]["inference_evidence_sha256"] = ( inference_descriptor["sha256"] ) raw_descriptor = _write_governed_json(repo_root, raw_path, raw_portfolio) evaluation = evaluate_cases(raw_path, set(evaluation_ids)) metric_report = { "schema_version": 3, "artifact_role": "protected_metric_report", "active_model_sha256": active_model["sha256"], "configuration_sha256": configuration_descriptor["sha256"], "evaluator_sha256": evaluator_hash, "protected_split_sha256": protected_descriptor["sha256"], "raw_predictions_sha256": raw_descriptor["sha256"], "evaluator_version": evaluation["evaluator_version"], "portfolio_kind": evaluation["portfolio_kind"], "portfolio_id": evaluation["portfolio_id"], "portfolio_file_sha256": evaluation["portfolio_file_sha256"], "portfolio_canonical_json_sha256": evaluation[ "portfolio_canonical_json_sha256" ], "evaluated_task_families": evaluation["evaluated_task_families"], "task_count": evaluation["task_count"], "case_count": evaluation["case_count"], "task_inventory": evaluation["task_inventory"], "task_inventory_sha256": canonical_hash(evaluation["task_inventory"]), "results": evaluation["results"], "results_canonical_json_sha256": canonical_hash(evaluation["results"]), "portfolio_metrics": evaluation["portfolio_metrics"], "portfolio_metrics_canonical_json_sha256": canonical_hash( evaluation["portfolio_metrics"] ), "subgroups": evaluation["subgroups"], "subgroups_canonical_json_sha256": canonical_hash(evaluation["subgroups"]), "failures": evaluation["failures"], "failures_canonical_json_sha256": canonical_hash(evaluation["failures"]), "failure_taxonomy": evaluation["failure_taxonomy"], "failure_taxonomy_canonical_json_sha256": canonical_hash( evaluation["failure_taxonomy"] ), "subgroup_dimension_mapping": REQUIRED_SUBGROUP_DIMENSION_FIELDS, "subgroup_release_policy": SUBGROUP_RELEASE_POLICY, "pre_registered_targets": subgroup_targets, "pre_registered_targets_sha256": canonical_hash(subgroup_targets), } metric_descriptor = _write_governed_json( repo_root, evidence_root / "metric-report.json", metric_report, ) review_entries = [] protected_by_id = {sample["sample_id"]: sample for sample in protected_samples} for sample_id in evaluation_ids: sample = protected_by_id[sample_id] base_entry = { "sample_id": sample_id, "reviewer_id": "human-reviewer-fixture", "review_timestamp": "2026-08-02T08:00:00+00:00", "decision": "accepted", "label_sha256": sample["label_sha256"], "case_input_sha256": sample["case_input_sha256"], } review_entries.append( { **base_entry, "entry_canonical_json_sha256": canonical_hash(base_entry), } ) review_ledger = { "schema_version": 1, "artifact_role": "human_review_ledger", "protected_split_sha256": protected_descriptor["sha256"], "raw_predictions_sha256": raw_descriptor["sha256"], "entries": review_entries, "entries_canonical_json_sha256": canonical_hash(review_entries), } review_descriptor = _write_governed_json( repo_root, evidence_root / "human-review-ledger.json", review_ledger, ) leakage_audit = { "schema_version": 1, "artifact_role": "geometric_leakage_audit", "protected_split_sha256": protected_descriptor["sha256"], "raw_predictions_sha256": raw_descriptor["sha256"], "development_split_manifest_sha256": development_split_hash, "distance_threshold_m": 2000.0, "projected_crs": "EPSG:31370", "algorithm": "projected_geometry_nearest_aoi_distance_v1", "evaluation_sample_ids_sha256": canonical_hash(evaluation_ids), "below_threshold_pair_count": 0, "below_threshold_pairs": [], "minimum_observed_distance_m": 2500.0, } leakage_descriptor = _write_governed_json( repo_root, evidence_root / "geometric-leakage-audit.json", leakage_audit, ) access_base = { "sequence": 1, "timestamp": "2026-08-02T10:00:00+00:00", "actor": "phase4-evaluator", "purpose": "evaluation_only", "operation": "read", "sample_ids": evaluation_ids, "previous_entry_sha256": "0" * 64, } access_entry = {**access_base, "entry_sha256": canonical_hash(access_base)} access_log = [access_entry] vault_evidence = { "schema_version": 1, "artifact_role": "vault_access_evidence", "protected_split_sha256": protected_descriptor["sha256"], "raw_predictions_sha256": raw_descriptor["sha256"], "execution_id": execution_id, "vault_mode": "read_only_evaluation", "access_log": access_log, "access_log_canonical_json_sha256": canonical_hash(access_log), "challenge_labels_accessed": False, } vault_descriptor = _write_governed_json( repo_root, evidence_root / "vault-access-evidence.json", vault_evidence, ) manifest = { "schema_version": 2, "manifest_type": "geointel_governed_product_baseline", "baseline_id": baseline_id, "created_at": "2026-08-02T10:06:00+00:00", "status": "pass", "synthetic": False, "active_model": { key: active_model[key] for key in ("model_id", "model_version", "sha256", "size_bytes") }, "active_model_sha256": active_model["sha256"], "evaluator_sha256": evaluator_hash, "configuration_sha256": configuration_descriptor["sha256"], "development_split_manifest_sha256": development_split_hash, "selection_isolation": { "test_used_for_training": False, "test_used_for_threshold_selection": False, "test_used_for_model_selection": False, "test_used_for_iterative_error_correction": False, "challenge_labels_accessed": False, "operating_point_frozen_before_protected_inference": True, "configuration_sha256": configuration_descriptor["sha256"], }, "inference": { "executed": True, "execution_id": execution_id, "device": "cuda:0", "test_used_for_selection": False, }, "configuration": configuration_descriptor, "protected_split_manifest": protected_descriptor, "authoritative_reference_manifest": authority_descriptor, "inference_evidence": inference_descriptor, "raw_predictions": raw_descriptor, "metric_report": metric_descriptor, "human_review_ledger": review_descriptor, "geometric_leakage_audit": leakage_descriptor, "vault_access_evidence": vault_descriptor, } manifest_path = evidence_root / "product-baseline-manifest.json" _write_governed_json(repo_root, manifest_path, manifest) return { "repo_root": repo_root, "manifest_path": manifest_path, "active_model": active_model, "runtime_observation": runtime_observation, "evaluation": evaluation, "evaluation_ids": evaluation_ids, } def _rewrite_governed_artifact( fixture: dict[str, Any], role: str, mutate: Any, ) -> None: repo_root = fixture["repo_root"] manifest_path = fixture["manifest_path"] manifest = json.loads(manifest_path.read_text(encoding="utf-8")) artifact_path = repo_root / manifest[role]["path"] payload = json.loads(artifact_path.read_text(encoding="utf-8")) mutate(payload) manifest[role] = _write_governed_json(repo_root, artifact_path, payload) _write_governed_json(repo_root, manifest_path, manifest) def load_source() -> dict: return json.loads(SOURCE.read_text(encoding="utf-8")) def evaluation_inputs() -> tuple[dict, dict, dict, dict]: source = load_source() development, protected, leakage = build_manifests(source, trusted_fixture_mode=True) split_result = { "development": development, "protected": protected, "leakage": leakage, } allowed = { item["sample_id"] for item in protected["samples"] if item["split"] in {"test", "background-test"} } evaluation = evaluate_cases(CASES, allowed) portfolio = json.loads(CASES.read_text(encoding="utf-8")) firewall = firewall_contract_checks(split_result, CASES) return split_result, evaluation, portfolio, firewall def _governed_baseline_gate(fixture: dict[str, Any]) -> dict[str, Any]: # Runtime is the only mocked part: the fixture tests evidence structure, not accuracy. return product_baseline_manifest_gate( fixture["repo_root"], fixture["manifest_path"], fixture["active_model"], [dict(item) for item in REQUIRED_AUTHORITY_REQUIREMENTS], runtime_probe=lambda: copy.deepcopy(fixture["runtime_observation"]), ) def test_governed_product_baseline_validator_accepts_evaluator_derived_fixture( tmp_path: Path, ) -> None: fixture = _build_governed_product_fixture(tmp_path) gate = _governed_baseline_gate(fixture) assert gate["status"] == "pass" assert gate["violations"] == [] assert len(gate["checked_artifacts"]) == 9 assert all( check["status"] == "pass" for check in gate["validation_checks"].values() ) assert fixture["evaluation"]["case_count"] == 70 assert set(fixture["evaluation"]["evaluated_task_families"]) == TASKS assert all( metrics["case_support"] == 10 for metrics in fixture["evaluation"]["portfolio_metrics"].values() ) for name in ( "authoritative_reference_portfolio_available", "human_review_complete", "split_independence", "protected_storage_isolation", "representative_product_subgroup_support", ): assert gate["derived_gates"][name]["status"] == "pass" @pytest.mark.parametrize( ("role", "mutation", "expected_violation"), [ ( "inference_evidence", "cuda_unavailable", "inference_evidence:torch_cuda_unavailable", ), ( "authoritative_reference_manifest", "authority_missing", "authoritative_reference:missing_requirement", ), ( "raw_predictions", "prediction_missing", "raw_predictions:", ), ( "metric_report", "subgroup_count", "metric_report:", ), ( "human_review_ledger", "review_missing", "human_review:", ), ( "geometric_leakage_audit", "close_pair", "geometric_leakage:", ), ( "vault_access_evidence", "training_access", "vault_access:", ), ], ) def test_governed_product_baseline_validator_rejects_semantic_tampering( tmp_path: Path, role: str, mutation: str, expected_violation: str, ) -> None: fixture = _build_governed_product_fixture(tmp_path) def mutate(payload: dict[str, Any]) -> None: if mutation == "cuda_unavailable": payload["torch_cuda_is_available"] = False elif mutation == "authority_missing": payload["entries"].pop() payload["entries_canonical_json_sha256"] = canonical_hash( payload["entries"] ) elif mutation == "prediction_missing": missing_task = sorted(TASKS)[0] payload["cases"] = [ case for case in payload["cases"] if case["task"] != missing_task ] elif mutation == "subgroup_count": payload["subgroups"]["dimensions"]["region"]["strata"]["flanders"][ "case_support" ] += 1 payload["subgroups_canonical_json_sha256"] = canonical_hash( payload["subgroups"] ) elif mutation == "review_missing": payload["entries"].pop() payload["entries_canonical_json_sha256"] = canonical_hash( payload["entries"] ) elif mutation == "close_pair": payload["below_threshold_pair_count"] = 1 payload["below_threshold_pairs"] = [ { "development_sample_id": "development-neighbour", "protected_sample_id": fixture["evaluation_ids"][0], "distance_m": 1999.0, } ] payload["minimum_observed_distance_m"] = 1999.0 elif mutation == "training_access": entry = payload["access_log"][0] entry["purpose"] = "training" unsigned = { key: value for key, value in entry.items() if key != "entry_sha256" } entry["entry_sha256"] = canonical_hash(unsigned) payload["access_log_canonical_json_sha256"] = canonical_hash( payload["access_log"] ) else: # pragma: no cover - parametrization owns this closed set raise AssertionError(mutation) _rewrite_governed_artifact(fixture, role, mutate) gate = _governed_baseline_gate(fixture) assert gate["status"] == "fail" assert any( violation.startswith(expected_violation) for violation in gate["violations"] ) def test_governed_validator_rejects_impossible_metric_before_hash_comparison( tmp_path: Path, ) -> None: fixture = _build_governed_product_fixture(tmp_path) def mutate(payload: dict[str, Any]) -> None: payload["results"][0]["metrics"]["precision"] = 1.5 payload["results_canonical_json_sha256"] = canonical_hash(payload["results"]) _rewrite_governed_artifact(fixture, "metric_report", mutate) gate = _governed_baseline_gate(fixture) assert gate["status"] == "fail" assert any( violation.startswith("metric_report:") and ("range" in violation or "impossible" in violation) for violation in gate["violations"] ) def test_governed_validator_rejects_all_empty_observation_support( tmp_path: Path, ) -> None: fixture = _build_governed_product_fixture(tmp_path) def mutate(payload: dict[str, Any]) -> None: for metrics in payload["portfolio_metrics"].values(): metrics["observation_support"] = { key: 0 for key in metrics["observation_support"] } payload["portfolio_metrics_canonical_json_sha256"] = canonical_hash( payload["portfolio_metrics"] ) _rewrite_governed_artifact(fixture, "metric_report", mutate) gate = _governed_baseline_gate(fixture) assert gate["status"] == "fail" assert any( violation.startswith("metric_report:") and "empty_support" in violation for violation in gate["violations"] ) def test_governed_validator_rejects_missing_evaluator_task_family( tmp_path: Path, ) -> None: fixture = _build_governed_product_fixture(tmp_path) missing_task = "terrain_interpretation" def mutate(payload: dict[str, Any]) -> None: payload["cases"] = [ case for case in payload["cases"] if case["task"] != missing_task ] _rewrite_governed_artifact(fixture, "raw_predictions", mutate) gate = _governed_baseline_gate(fixture) assert gate["status"] == "fail" assert any( violation.startswith("raw_predictions:") and ("task" in violation or "evaluator" in violation) for violation in gate["violations"] ) def test_governed_product_baseline_validator_rejects_manifest_claim_tampering( tmp_path: Path, ) -> None: fixture = _build_governed_product_fixture(tmp_path) manifest = json.loads(fixture["manifest_path"].read_text(encoding="utf-8")) manifest["synthetic"] = True _write_governed_json( fixture["repo_root"], fixture["manifest_path"], manifest, ) gate = _governed_baseline_gate(fixture) assert gate["status"] == "fail" assert "manifest:synthetic_or_unspecified" in gate["violations"] def test_active_model_checksum_mismatch_is_explicit_failure(tmp_path: Path) -> None: model_path = tmp_path / "active.pt" model_path.write_bytes(b"observed-model") gate = active_model_availability_gate( { "path": str(model_path), "sha256": _fixture_hash("different-model"), "size_bytes": model_path.stat().st_size, } ) assert gate["status"] == "fail" assert gate["observed_sha256"] != gate["configured_sha256"] assert "checksum" in gate["reason"].lower() def test_missing_product_gates_fail_closed_and_explicit_fail_has_precedence() -> None: split_result, evaluation, portfolio, firewall = evaluation_inputs() missing_report = build_release_gate_report( split_result, evaluation, portfolio, canonical_golden_baseline(), firewall, {}, ) assert missing_report["status"] == "fail" assert missing_report["product_benchmark_status"] == "fail" assert set(missing_report["missing_gate_names"]["product"]) == PRODUCT_GATE_NAMES product_gates = {name: {"status": "pass"} for name in PRODUCT_GATE_NAMES} product_gates["executed_product_incumbent_baseline"] = {"status": "not_evaluable"} product_gates["human_review_complete"] = {"status": "fail"} precedence_report = build_release_gate_report( split_result, evaluation, portfolio, canonical_golden_baseline(), firewall, product_gates, ) assert precedence_report["status"] == "fail" assert precedence_report["product_benchmark_status"] == "fail" def test_document_status_booleans_cannot_spoof_review_split_or_vault( tmp_path: Path, ) -> None: fixture = _build_governed_product_fixture(tmp_path) snapshot = { "active_model": fixture["active_model"], "authority_requirements": [ dict(item) for item in REQUIRED_AUTHORITY_REQUIREMENTS ], "v56_review_and_split": { "review_complete": True, "reviewed_sample_count": 999, "sample_count": 999, "split_independence_proven": True, "cross_split_pairs_below_2000_m": 0, }, "protected_test_isolation": True, "phase3_leakage_status": "pass", } missing_manifest = ( fixture["repo_root"] / "artifacts/evidence/accuracy/P4/missing/product-baseline-manifest.json" ) gates = product_gate_evidence(fixture["repo_root"], snapshot, missing_manifest) for name in ( "human_review_complete", "split_independence", "protected_storage_isolation", ): assert gates[name]["status"] in {"fail", "not_evaluable"} assert gates[name]["status"] != "pass" def test_all_mandatory_product_gates_make_phase5_reachable() -> None: split_result, evaluation, portfolio, firewall = evaluation_inputs() product_gates = {name: {"status": "pass"} for name in PRODUCT_GATE_NAMES} report = build_release_gate_report( split_result, evaluation, portfolio, canonical_golden_baseline(), firewall, product_gates, ) assert report["status"] == "pass" assert report["phase_decision"] == "ready_for_phase5" assert report["missing_gate_names"] == {"local": [], "product": []} assert set( report["local_gates"]["normative_split_roles_and_leakage"]["required_roles"] ) == {"train", "val", "calibration", "test", "background-test", "challenge"} def test_split_fixture_is_order_independent_and_has_all_roles() -> None: source = load_source() development, protected, leakage = build_manifests(source, trusted_fixture_mode=True) reversed_source = copy.deepcopy(source) reversed_source["samples"].reverse() reversed_development, reversed_protected, reversed_leakage = build_manifests( reversed_source, trusted_fixture_mode=True ) assert leakage["status"] == "pass" assert leakage["finding_count"] == 0 assert leakage["split_counts"] == { "background-test": 2, "calibration": 2, "challenge": 4, "test": 7, "train": 3, "val": 3, } assert reversed_development["manifest_sha256"] == development["manifest_sha256"] assert reversed_protected["manifest_sha256"] == protected["manifest_sha256"] assert reversed_leakage == leakage def test_training_firewall_rejects_non_train_and_protected_lineage() -> None: development, protected, leakage = build_manifests( load_source(), trusted_fixture_mode=True ) assert leakage["status"] == "pass" train = [item for item in development["samples"] if item["split"] == "train"] validation = next(item for item in development["samples"] if item["split"] == "val") assert_training_inputs_safe([], train, protected, trusted_fixture_mode=True) with pytest.raises(LeakageError, match="non_train_role"): assert_training_inputs_safe( [], [validation], protected, trusted_fixture_mode=True ) with pytest.raises(LeakageError, match="protected_identity"): disguised = dict(train[0]) disguised["source_family"] = protected["samples"][0]["source_family"] assert_training_inputs_safe( [], [disguised], protected, trusted_fixture_mode=True ) with pytest.raises(LeakageError, match="protected_path"): assert_training_inputs_safe([CASES], [], protected, trusted_fixture_mode=True) def test_task_evaluator_retains_exact_raw_inputs_metrics_and_failures() -> None: _split_result, report, _portfolio, _firewall = evaluation_inputs() assert report["task_count"] == 7 assert report["case_count"] == 9 assert len(report["task_inventory"]) >= 15 assert len(report["failures"]) >= 11 assert report["subgroups"]["overall_status"] == "not_evaluable" assert all( { "references", "predictions_pre_filter", "predictions_post_filter", "config", "input_lineage", "portfolio_lineage", } <= set(item["raw"]) for item in report["results"] ) detection = next( item for item in report["results"] if item["sample_id"] == "det-test-flanders-urban" ) assert len(detection["raw"]["predictions_pre_filter"]) == 4 assert len(detection["raw"]["predictions_post_filter"]) == 3 assert detection["metrics"]["true_positive"] == 2 assert detection["metrics"]["false_positive"] == 1 assert detection["metrics"]["ap50"] is not None empty = next( item for item in report["results"] if item["sample_id"] == "background-test-pure-empty" ) assert empty["metrics"]["precision"] is None assert empty["metrics"]["recall"] is None assert empty["metrics"]["f1"] is None def test_product_prerequisites_cannot_pass_without_executed_baseline() -> None: split_result, evaluation, portfolio, firewall = evaluation_inputs() product_gates = { "active_model_available_and_hash_verified": {"status": "pass"}, "authoritative_reference_portfolio_available": {"status": "pass"}, "human_review_complete": {"status": "pass"}, "split_independence": {"status": "pass"}, "phase3_leakage_resolved": {"status": "pass"}, "protected_storage_isolation": {"status": "pass"}, "executed_product_incumbent_baseline": { "status": "not_evaluable", "reason": "no raw active-model inference", }, "representative_product_subgroup_support": {"status": "pass"}, } report = build_release_gate_report( split_result, evaluation, portfolio, canonical_golden_baseline(), firewall, product_gates, ) assert report["local_harness_status"] == "pass" assert report["product_benchmark_status"] == "not_evaluable" assert report["status"] == "not_evaluable" assert report["phase_decision"] == "blocked" def test_one_workflow_is_byte_reproducible_complete_and_fail_closed( tmp_path: Path, ) -> None: output = tmp_path / "p4" first = run_workflow(ROOT, output) first_bytes = {path.name: path.read_bytes() for path in output.glob("*.json")} second = run_workflow(ROOT, output) second_bytes = {path.name: path.read_bytes() for path in output.glob("*.json")} assert first == second assert first_bytes == second_bytes assert first["local_harness_status"] == "pass" assert first["product_benchmark_status"] == "fail" assert first["status"] == "fail" assert first["phase4_done"] is False assert first["phase5_ready"] is False required = { "acceptance-gates.json", "aoi-metrics.json", "baseline-raw-predictions.json", "benchmark-manifest.json", "calibration-metrics.json", "candidate-vs-incumbent.json", "development-split-manifest.json", "error-taxonomy.json", "evaluation-contract.json", "failure-gallery.json", "generation-status.json", "human-review-summary.json", "input-manifest.json", "latency-and-reliability.json", "leakage-gate-report.json", "metric-report.json", "object-metrics.json", "protected-split-manifest.json", "reference-implementation-baseline.json", "release-gate-report.json", "split-and-leakage-audit.json", "stratified-metrics.json", "tile-metrics.json", "workflow-summary.json", "evidence-manifest.json", } assert required == set(first_bytes) manifest = json.loads( (output / "evidence-manifest.json").read_text(encoding="utf-8") ) assert manifest["artifact_count"] == len(required) - 1 for item in manifest["artifacts"]: path = output / item["path"] assert path.stat().st_size == item["size_bytes"] assert hashlib.sha256(path.read_bytes()).hexdigest() == item["sha256"] gates = json.loads( (output / "release-gate-report.json").read_text(encoding="utf-8") ) assert gates["promotion_allowed"] is False assert all(item["status"] == "pass" for item in gates["local_gates"].values()) assert {item["status"] for item in gates["product_gates"].values()} <= { "pass", "fail", "not_evaluable", } input_manifest = json.loads( (output / "input-manifest.json").read_text(encoding="utf-8") ) benchmark_manifest = json.loads( (output / "benchmark-manifest.json").read_text(encoding="utf-8") ) assert input_manifest["product_baseline"]["validation_status"] == "not_evaluable" assert input_manifest["product_baseline"]["artifacts"] == [] assert "docs/accuracy-program/status.json" not in { item["path"] for item in input_manifest["inputs"] } assert input_manifest["readiness_snapshot"]["source_paths"][ "accuracy_status_projection" ]["selected_json_pointers"] == ["/runtime/active_model"] assert benchmark_manifest["product_baseline"] == input_manifest["product_baseline"] assert ( benchmark_manifest["product_gate_evidence_sha256"] == first["product_gate_evidence_sha256"] ) assert first["evidence_run_id"].startswith("p4-2.0.1-") def test_readiness_snapshot_ignores_phase4_bookkeeping_but_binds_active_model( tmp_path: Path, ) -> None: status_path = tmp_path / "docs/accuracy-program/status.json" scan_path = tmp_path / "artifacts/evidence/accuracy/P3/full-scan-manifest.json" leakage_path = tmp_path / "artifacts/evidence/accuracy/P3/leakage-report.json" status_path.parent.mkdir(parents=True) scan_path.parent.mkdir(parents=True) status = { "generated_at": "2026-08-02T00:00:00+02:00", "documents": ["old.md"], "phase5": {"status": "not_ready"}, "verification": {"phase4_evaluation": {"status": "pending"}}, "phase4": {"status": "ready"}, "runtime": { "active_model": { "model_id": "model-a", "path": "/models/a.pt", "sha256": "a" * 64, } }, } status_path.write_text(json.dumps(status), encoding="utf-8") scan_path.write_text( json.dumps({"scan_id": "scan-a", "content_hash": "b" * 64}), encoding="utf-8", ) leakage_path.write_text(json.dumps({"status": "attention"}), encoding="utf-8") original = readiness_snapshot(tmp_path) status["phase4"] = {"status": "in_progress", "evidence_run_id": "run-a"} status["generated_at"] = "2026-08-02T05:00:00+02:00" status["documents"] = ["old.md", "new.md"] status["phase5"] = {"status": "blocked"} status["verification"] = {"phase4_evaluation": {"status": "local_pass"}} status_path.write_text(json.dumps(status), encoding="utf-8") bookkeeping_update = readiness_snapshot(tmp_path) assert bookkeeping_update == original assert "accuracy_status" not in original["source_paths"] assert original["source_paths"]["accuracy_status_projection"][ "selected_json_pointers" ] == ["/runtime/active_model"] status["runtime"]["active_model"]["sha256"] = "c" * 64 status_path.write_text(json.dumps(status), encoding="utf-8") assert readiness_snapshot(tmp_path) != original assert original["source_paths"]["accuracy_status_projection"][ "sha256" ] == canonical_hash( { "schema_version": 1, "runtime": {"active_model": original["active_model"]}, } ) def test_immutable_workflow_refuses_to_replace_changed_evidence(tmp_path: Path) -> None: output = tmp_path / "p4" run_workflow(ROOT, output) (output / "workflow-summary.json").write_text("{}\n", encoding="utf-8") with pytest.raises(EvidenceConflictError, match="Refusing to overwrite"): run_workflow(ROOT, output) def test_immutable_workflow_rejects_rogue_nested_evidence(tmp_path: Path) -> None: output = tmp_path / "p4" run_workflow(ROOT, output) rogue = output / "rogue" / "unmanifested.json" rogue.parent.mkdir() rogue.write_text("{}\n", encoding="utf-8") with pytest.raises(EvidenceConflictError, match="(?i)unexpected|immutable"): run_workflow(ROOT, output)