From ee1982eff28609ff870527cce05d1c9059a83945 Mon Sep 17 00:00:00 2001 From: Jens Date: Sun, 2 Aug 2026 04:29:10 +0200 Subject: [PATCH] fix(accuracy): close phase 4 evidence bypasses --- .../tests/test_accuracy_phase4_evaluation.py | 993 ++++++- ...est_accuracy_phase4_evaluator_hardening.py | 331 ++- .../test_accuracy_phase4_split_hardening.py | 354 ++- .../accuracy/p4/protected-baseline-cases.json | 1 + scripts/accuracy_phase4_evaluator.py | 642 ++++- scripts/generate_accuracy_phase4_splits.py | 1417 +++++++++- scripts/run_accuracy_phase4_benchmark.py | 2458 +++++++++++++++-- 7 files changed, 5814 insertions(+), 382 deletions(-) diff --git a/backend/tests/test_accuracy_phase4_evaluation.py b/backend/tests/test_accuracy_phase4_evaluation.py index d65561b1..bf8c21bf 100644 --- a/backend/tests/test_accuracy_phase4_evaluation.py +++ b/backend/tests/test_accuracy_phase4_evaluation.py @@ -5,6 +5,7 @@ import hashlib import json import sys from pathlib import Path +from typing import Any import pytest @@ -14,7 +15,12 @@ SCRIPTS = ROOT / "scripts" if str(SCRIPTS) not in sys.path: sys.path.insert(0, str(SCRIPTS)) -from accuracy_phase4_evaluator import evaluate_cases # noqa: E402 +from accuracy_phase4_evaluator import ( # noqa: E402 + TASKS, + canonical_hash, + evaluate_cases, + task_inventory, +) from generate_accuracy_phase4_splits import ( # noqa: E402 LeakageError, assert_training_inputs_safe, @@ -22,9 +28,16 @@ from generate_accuracy_phase4_splits import ( # noqa: E402 ) from run_accuracy_phase4_benchmark import ( # noqa: E402 EvidenceConflictError, + PRODUCT_GATE_NAMES, + REQUIRED_AUTHORITY_REQUIREMENTS, + REQUIRED_SUBGROUP_DIMENSION_FIELDS, + SUBGROUP_RELEASE_POLICY, + active_model_availability_gate, build_release_gate_report, canonical_golden_baseline, firewall_contract_checks, + product_baseline_manifest_gate, + product_gate_evidence, run_workflow, ) @@ -33,13 +46,607 @@ SOURCE = ROOT / "fixtures/accuracy/p4/split-source-manifest.json" CASES = ROOT / "fixtures/accuracy/p4/protected-baseline-cases.json" +def _fixture_hash(value: str) -> str: + return hashlib.sha256(value.encode("utf-8")).hexdigest() + + +def _write_governed_json( + repo_root: Path, + path: Path, + payload: dict[str, Any], +) -> dict[str, Any]: + path.parent.mkdir(parents=True, exist_ok=True) + content = ( + json.dumps(payload, ensure_ascii=False, indent=2, sort_keys=True) + "\n" + ).encode("utf-8") + path.write_bytes(content) + return { + "path": path.relative_to(repo_root).as_posix(), + "sha256": hashlib.sha256(content).hexdigest(), + "size_bytes": len(content), + } + + +def _build_governed_product_fixture(tmp_path: Path) -> dict[str, Any]: + """Build structural governance evidence; this fixture makes no accuracy claim.""" + + repo_root = tmp_path / "repo" + evaluator_path = repo_root / "scripts/accuracy_phase4_evaluator.py" + evaluator_path.parent.mkdir(parents=True, exist_ok=True) + evaluator_path.write_text("# governed evaluator fixture\n", encoding="utf-8") + evaluator_hash = hashlib.sha256(evaluator_path.read_bytes()).hexdigest() + + model_path = repo_root / "models/active.pt" + model_path.parent.mkdir(parents=True, exist_ok=True) + model_path.write_bytes(b"governed-model-fixture") + active_model = { + "model_id": "fixture-model", + "model_version": "1.0.0", + "path": str(model_path), + "sha256": hashlib.sha256(model_path.read_bytes()).hexdigest(), + "size_bytes": model_path.stat().st_size, + } + baseline_id = "governed-product-fixture" + evidence_root = repo_root / "artifacts/evidence/accuracy/P4/governed-fixture" + raw_path = evidence_root / "raw-predictions.json" + development_split_hash = _fixture_hash("governed-development-split-v1") + + source_portfolio = json.loads(CASES.read_text(encoding="utf-8")) + templates: dict[str, dict[str, Any]] = {} + for case in source_portfolio["cases"]: + templates.setdefault(case["task"], case) + assert set(templates) == TASKS + + profiles = ( + { + "region": "flanders", + "municipality": "Mol", + "urbanity": "urban", + "object_size": "small", + "source": "governed-grb-orthophoto", + "sensor": "aerial-rgb", + "resolution_m": 0.25, + "season": "summer", + "date": "2025-06-15", + "vegetation": "low", + "occlusion": "none", + "difficulty": "normal", + "context": "dense_urban", + }, + { + "region": "wallonia", + "municipality": "Namur", + "urbanity": "rural", + "object_size": "large", + "source": "governed-picc-orthophoto", + "sensor": "multispectral-rgb", + "resolution_m": 1.0, + "season": "winter", + "date": "2025-01-15", + "vegetation": "high", + "occlusion": "partial", + "difficulty": "hard", + "context": "rural_occluded", + }, + ) + authority_scopes = [ + { + "task": requirement["task"], + "zone": requirement["zone"], + "authority": requirement["primary"], + } + for requirement in REQUIRED_AUTHORITY_REQUIREMENTS + ] + + cases: list[dict[str, Any]] = [] + protected_samples: list[dict[str, Any]] = [] + parameters_by_task = { + task: copy.deepcopy(templates[task]["config"]) for task in sorted(TASKS) + } + for task in sorted(TASKS): + for profile_index, profile in enumerate(profiles): + stratum = "a" if profile_index == 0 else "b" + for replicate in range(5): + sample_id = f"governed-{task}-{stratum}-{replicate}" + case = copy.deepcopy(templates[task]) + case["sample_id"] = sample_id + case["split"] = ( + "background-test" + if task == "object_detection" and profile_index == 1 + else "test" + ) + case["metadata"].update(profile) + case["metadata"].update( + { + "tile_edge": profile_index == 1, + "label_review_state": "human_reviewed_fixture", + "ood": False, + } + ) + case["config"] = copy.deepcopy(parameters_by_task[task]) + case["lineage"] = { + "reference": { + "source_id": f"governed:{sample_id}:reference", + "source_version": "fixture-v1", + "derivation": "structural_contract_fixture_reference", + }, + "prediction": { + "source_id": f"governed:{sample_id}:prediction", + "source_version": "fixture-v1", + "derivation": "structural_contract_fixture_prediction", + }, + } + cases.append(case) + reference_payload = ( + case["expected_anomalies"] + if task == "geospatial_data_validation" + else case["references"] + ) + subgroups = { + dimension: case["metadata"][metadata_field] + for dimension, metadata_field in ( + ("region", "region"), + ("municipality", "municipality"), + ("urbanity", "urbanity"), + ("object_size", "object_size"), + ("source", "source"), + ("sensor", "sensor"), + ("resolution", "resolution_m"), + ("season", "season"), + ("date", "date"), + ("vegetation", "vegetation"), + ("occlusion", "occlusion"), + ("difficulty", "difficulty"), + ("context", "context"), + ) + } + protected_samples.append( + { + "sample_id": sample_id, + "split": case["split"], + "task": task, + "zone": str(profile["region"]), + "aoi_id": f"independent-aoi-{task}-{stratum}-{replicate}", + "content_sha256": _fixture_hash(f"content:{sample_id}"), + "label_sha256": canonical_hash(reference_payload), + "case_input_sha256": canonical_hash(case), + "labels_access_policy": "evaluation_only", + "subgroups": subgroups, + "authority_scopes": copy.deepcopy(authority_scopes), + } + ) + + challenge_id = "governed-challenge-sealed" + protected_samples.append( + { + "sample_id": challenge_id, + "split": "challenge", + "task": "object_detection", + "zone": "flanders", + "aoi_id": "independent-aoi-challenge-sealed", + "content_sha256": _fixture_hash(f"content:{challenge_id}"), + "labels_sealed": True, + "subgroups": { + dimension: profiles[0][metadata_field] + for dimension, metadata_field in ( + ("region", "region"), + ("municipality", "municipality"), + ("urbanity", "urbanity"), + ("object_size", "object_size"), + ("source", "source"), + ("sensor", "sensor"), + ("resolution", "resolution_m"), + ("season", "season"), + ("date", "date"), + ("vegetation", "vegetation"), + ("occlusion", "occlusion"), + ("difficulty", "difficulty"), + ("context", "context"), + ) + }, + "authority_scopes": copy.deepcopy(authority_scopes), + } + ) + evaluation_ids = sorted(case["sample_id"] for case in cases) + all_ids = sorted(sample["sample_id"] for sample in protected_samples) + task_sample_ids = { + task: sorted(case["sample_id"] for case in cases if case["task"] == task) + for task in sorted(TASKS) + } + split_counts = { + split: sum(sample["split"] == split for sample in protected_samples) + for split in sorted({sample["split"] for sample in protected_samples}) + } + protected_split = { + "schema_version": 3, + "artifact_role": "protected_evaluation_split", + "protected_policy": { + "immutable": True, + "training_allowed": False, + "threshold_selection_allowed": False, + "model_selection_allowed": False, + "iterative_error_correction_allowed": False, + "challenge_labels_accessible": False, + }, + "evaluator_task_inventory_sha256": canonical_hash(task_inventory()), + "samples": protected_samples, + "split_counts": split_counts, + "sample_ids_sha256": canonical_hash(all_ids), + "evaluation_sample_ids_sha256": canonical_hash(evaluation_ids), + "evaluated_task_families": sorted(TASKS), + "task_sample_ids": task_sample_ids, + "task_sample_ids_canonical_json_sha256": canonical_hash(task_sample_ids), + } + protected_descriptor = _write_governed_json( + repo_root, + evidence_root / "protected-split.json", + protected_split, + ) + + sample_references = [ + { + "sample_id": case["sample_id"], + "task": case["task"], + "reference_payload_sha256": canonical_hash( + case["expected_anomalies"] + if case["task"] == "geospatial_data_validation" + else case["references"] + ), + "reference_lineage_sha256": canonical_hash(case["lineage"]["reference"]), + } + for case in sorted(cases, key=lambda item: item["sample_id"]) + ] + authority_entries = [ + { + "task": requirement["task"], + "zone": requirement["zone"], + "authority": requirement["primary"], + "source_classification": "authoritative", + "source_snapshot_id": f"snapshot-{requirement['primary']}-2026", + "source_snapshot_sha256": _fixture_hash( + f"snapshot:{requirement['primary']}" + ), + "sample_ids": evaluation_ids, + } + for requirement in REQUIRED_AUTHORITY_REQUIREMENTS + ] + authority_portfolio = { + "schema_version": 2, + "artifact_role": "authoritative_reference_portfolio", + "portfolio_id": "governed-authority-fixture", + "protected_split_sha256": protected_descriptor["sha256"], + "entries": authority_entries, + "entries_canonical_json_sha256": canonical_hash(authority_entries), + "sample_references": sample_references, + "sample_references_canonical_json_sha256": canonical_hash(sample_references), + } + authority_descriptor = _write_governed_json( + repo_root, + evidence_root / "authoritative-reference.json", + authority_portfolio, + ) + + portfolio_lineage = { + "origin": "governed_product_inference", + "source_path": raw_path.relative_to(repo_root).as_posix(), + "version": baseline_id, + "active_model_sha256": active_model["sha256"], + "configuration_sha256": "0" * 64, + "protected_split_sha256": protected_descriptor["sha256"], + "authoritative_reference_sha256": authority_descriptor["sha256"], + "inference_evidence_sha256": "0" * 64, + } + raw_portfolio = { + "schema_version": 2, + "portfolio_kind": "governed_product_baseline", + "portfolio_id": baseline_id, + "split_roles": ["test", "background-test"], + "selection_policy": "frozen_validation_calibration_only_no_protected_selection", + "protected_policy": { + "operating_point_selection_allowed": False, + "diagnostic_curves_select_operating_point": False, + "test_feedback_allowed": False, + "threshold_selection_source": "pre_registered_configuration_only", + }, + "portfolio_lineage": portfolio_lineage, + "claim_boundary": ( + "Governed product baseline structural fixture validates evidence " + "contracts only and makes no production accuracy claim." + ), + "cases": cases, + } + preliminary_path = repo_root / "preliminary-product-portfolio.json" + _write_governed_json(repo_root, preliminary_path, raw_portfolio) + preliminary_evaluation = evaluate_cases(preliminary_path, set(evaluation_ids)) + preliminary_path.unlink() + subgroup_targets = { + task: { + "metric": metrics["primary_metric"]["name"], + "direction": metrics["primary_metric"]["direction"], + "threshold": metrics["primary_metric"]["value"], + } + for task, metrics in preliminary_evaluation["portfolio_metrics"].items() + } + + configuration = { + "schema_version": 2, + "artifact_role": "frozen_inference_configuration", + "active_model_sha256": active_model["sha256"], + "development_split_manifest_sha256": development_split_hash, + "frozen_before_protected_access": True, + "frozen_at": "2026-08-02T09:00:00+00:00", + "protected_data_used": False, + "threshold_selection_source": "validation_and_calibration", + "parameters_by_task": parameters_by_task, + "subgroup_release_policy": SUBGROUP_RELEASE_POLICY, + "subgroup_release_targets": subgroup_targets, + "subgroup_release_targets_sha256": canonical_hash(subgroup_targets), + } + configuration_descriptor = _write_governed_json( + repo_root, + evidence_root / "configuration.json", + configuration, + ) + + execution_id = "cuda-execution-fixture-001" + runtime_observation = { + "status": "pass", + "device": "cuda:0", + "device_name": "NVIDIA governed fixture", + "gpu_uuid": "GPU-governed-fixture", + "driver_version": "570.00", + "cuda_runtime_version": "12.8", + "torch_version": "2.7.0", + "cuda_device_count": 1, + "kernel_execution_confirmed": True, + } + inference_evidence = { + "schema_version": 2, + "artifact_role": "governed_cuda_inference_execution", + "execution_id": execution_id, + "active_model_sha256": active_model["sha256"], + "configuration_sha256": configuration_descriptor["sha256"], + "evaluator_sha256": evaluator_hash, + "protected_split_sha256": protected_descriptor["sha256"], + "executed": True, + "exit_code": 0, + "test_used_for_selection": False, + "device_type": "cuda", + "device": "cuda:0", + "torch_cuda_is_available": True, + "cuda_device_count": 1, + "kernel_execution_confirmed": True, + "batch_failure_count": 0, + "torch_version": "2.7.0", + "cuda_runtime_version": "12.8", + "driver_version": "570.00", + "started_at": "2026-08-02T10:00:00+00:00", + "finished_at": "2026-08-02T10:05:00+00:00", + "nvidia_smi": { + "gpu_uuid": "GPU-governed-fixture", + "device_name": "NVIDIA governed fixture", + "driver_version": "570.00", + "cuda_version": "12.8", + "query_output_sha256": _fixture_hash("nvidia-smi-output"), + }, + "processed_sample_ids": evaluation_ids, + "processed_sample_ids_sha256": canonical_hash(evaluation_ids), + "successful_sample_count": len(evaluation_ids), + } + inference_descriptor = _write_governed_json( + repo_root, + evidence_root / "inference-evidence.json", + inference_evidence, + ) + + raw_portfolio["portfolio_lineage"]["configuration_sha256"] = ( + configuration_descriptor["sha256"] + ) + raw_portfolio["portfolio_lineage"]["inference_evidence_sha256"] = ( + inference_descriptor["sha256"] + ) + raw_descriptor = _write_governed_json(repo_root, raw_path, raw_portfolio) + evaluation = evaluate_cases(raw_path, set(evaluation_ids)) + + metric_report = { + "schema_version": 3, + "artifact_role": "protected_metric_report", + "active_model_sha256": active_model["sha256"], + "configuration_sha256": configuration_descriptor["sha256"], + "evaluator_sha256": evaluator_hash, + "protected_split_sha256": protected_descriptor["sha256"], + "raw_predictions_sha256": raw_descriptor["sha256"], + "evaluator_version": evaluation["evaluator_version"], + "portfolio_kind": evaluation["portfolio_kind"], + "portfolio_id": evaluation["portfolio_id"], + "portfolio_file_sha256": evaluation["portfolio_file_sha256"], + "portfolio_canonical_json_sha256": evaluation[ + "portfolio_canonical_json_sha256" + ], + "evaluated_task_families": evaluation["evaluated_task_families"], + "task_count": evaluation["task_count"], + "case_count": evaluation["case_count"], + "task_inventory": evaluation["task_inventory"], + "task_inventory_sha256": canonical_hash(evaluation["task_inventory"]), + "results": evaluation["results"], + "results_canonical_json_sha256": canonical_hash(evaluation["results"]), + "portfolio_metrics": evaluation["portfolio_metrics"], + "portfolio_metrics_canonical_json_sha256": canonical_hash( + evaluation["portfolio_metrics"] + ), + "subgroups": evaluation["subgroups"], + "subgroups_canonical_json_sha256": canonical_hash(evaluation["subgroups"]), + "failures": evaluation["failures"], + "failures_canonical_json_sha256": canonical_hash(evaluation["failures"]), + "failure_taxonomy": evaluation["failure_taxonomy"], + "failure_taxonomy_canonical_json_sha256": canonical_hash( + evaluation["failure_taxonomy"] + ), + "subgroup_dimension_mapping": REQUIRED_SUBGROUP_DIMENSION_FIELDS, + "subgroup_release_policy": SUBGROUP_RELEASE_POLICY, + "pre_registered_targets": subgroup_targets, + "pre_registered_targets_sha256": canonical_hash(subgroup_targets), + } + metric_descriptor = _write_governed_json( + repo_root, + evidence_root / "metric-report.json", + metric_report, + ) + + review_entries = [] + protected_by_id = {sample["sample_id"]: sample for sample in protected_samples} + for sample_id in evaluation_ids: + sample = protected_by_id[sample_id] + base_entry = { + "sample_id": sample_id, + "reviewer_id": "human-reviewer-fixture", + "review_timestamp": "2026-08-02T08:00:00+00:00", + "decision": "accepted", + "label_sha256": sample["label_sha256"], + "case_input_sha256": sample["case_input_sha256"], + } + review_entries.append( + { + **base_entry, + "entry_canonical_json_sha256": canonical_hash(base_entry), + } + ) + review_ledger = { + "schema_version": 1, + "artifact_role": "human_review_ledger", + "protected_split_sha256": protected_descriptor["sha256"], + "raw_predictions_sha256": raw_descriptor["sha256"], + "entries": review_entries, + "entries_canonical_json_sha256": canonical_hash(review_entries), + } + review_descriptor = _write_governed_json( + repo_root, + evidence_root / "human-review-ledger.json", + review_ledger, + ) + + leakage_audit = { + "schema_version": 1, + "artifact_role": "geometric_leakage_audit", + "protected_split_sha256": protected_descriptor["sha256"], + "raw_predictions_sha256": raw_descriptor["sha256"], + "development_split_manifest_sha256": development_split_hash, + "distance_threshold_m": 2000.0, + "projected_crs": "EPSG:31370", + "algorithm": "projected_geometry_nearest_aoi_distance_v1", + "evaluation_sample_ids_sha256": canonical_hash(evaluation_ids), + "below_threshold_pair_count": 0, + "below_threshold_pairs": [], + "minimum_observed_distance_m": 2500.0, + } + leakage_descriptor = _write_governed_json( + repo_root, + evidence_root / "geometric-leakage-audit.json", + leakage_audit, + ) + + access_base = { + "sequence": 1, + "timestamp": "2026-08-02T10:00:00+00:00", + "actor": "phase4-evaluator", + "purpose": "evaluation_only", + "operation": "read", + "sample_ids": evaluation_ids, + "previous_entry_sha256": "0" * 64, + } + access_entry = {**access_base, "entry_sha256": canonical_hash(access_base)} + access_log = [access_entry] + vault_evidence = { + "schema_version": 1, + "artifact_role": "vault_access_evidence", + "protected_split_sha256": protected_descriptor["sha256"], + "raw_predictions_sha256": raw_descriptor["sha256"], + "execution_id": execution_id, + "vault_mode": "read_only_evaluation", + "access_log": access_log, + "access_log_canonical_json_sha256": canonical_hash(access_log), + "challenge_labels_accessed": False, + } + vault_descriptor = _write_governed_json( + repo_root, + evidence_root / "vault-access-evidence.json", + vault_evidence, + ) + + manifest = { + "schema_version": 2, + "manifest_type": "geointel_governed_product_baseline", + "baseline_id": baseline_id, + "created_at": "2026-08-02T10:06:00+00:00", + "status": "pass", + "synthetic": False, + "active_model": { + key: active_model[key] + for key in ("model_id", "model_version", "sha256", "size_bytes") + }, + "active_model_sha256": active_model["sha256"], + "evaluator_sha256": evaluator_hash, + "configuration_sha256": configuration_descriptor["sha256"], + "development_split_manifest_sha256": development_split_hash, + "selection_isolation": { + "test_used_for_training": False, + "test_used_for_threshold_selection": False, + "test_used_for_model_selection": False, + "test_used_for_iterative_error_correction": False, + "challenge_labels_accessed": False, + "operating_point_frozen_before_protected_inference": True, + "configuration_sha256": configuration_descriptor["sha256"], + }, + "inference": { + "executed": True, + "execution_id": execution_id, + "device": "cuda:0", + "test_used_for_selection": False, + }, + "configuration": configuration_descriptor, + "protected_split_manifest": protected_descriptor, + "authoritative_reference_manifest": authority_descriptor, + "inference_evidence": inference_descriptor, + "raw_predictions": raw_descriptor, + "metric_report": metric_descriptor, + "human_review_ledger": review_descriptor, + "geometric_leakage_audit": leakage_descriptor, + "vault_access_evidence": vault_descriptor, + } + manifest_path = evidence_root / "product-baseline-manifest.json" + _write_governed_json(repo_root, manifest_path, manifest) + return { + "repo_root": repo_root, + "manifest_path": manifest_path, + "active_model": active_model, + "runtime_observation": runtime_observation, + "evaluation": evaluation, + "evaluation_ids": evaluation_ids, + } + + +def _rewrite_governed_artifact( + fixture: dict[str, Any], + role: str, + mutate: Any, +) -> None: + repo_root = fixture["repo_root"] + manifest_path = fixture["manifest_path"] + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + artifact_path = repo_root / manifest[role]["path"] + payload = json.loads(artifact_path.read_text(encoding="utf-8")) + mutate(payload) + manifest[role] = _write_governed_json(repo_root, artifact_path, payload) + _write_governed_json(repo_root, manifest_path, manifest) + + def load_source() -> dict: return json.loads(SOURCE.read_text(encoding="utf-8")) def evaluation_inputs() -> tuple[dict, dict, dict, dict]: source = load_source() - development, protected, leakage = build_manifests(source) + development, protected, leakage = build_manifests(source, trusted_fixture_mode=True) split_result = { "development": development, "protected": protected, @@ -56,13 +663,343 @@ def evaluation_inputs() -> tuple[dict, dict, dict, dict]: return split_result, evaluation, portfolio, firewall +def _governed_baseline_gate(fixture: dict[str, Any]) -> dict[str, Any]: + # Runtime is the only mocked part: the fixture tests evidence structure, not accuracy. + return product_baseline_manifest_gate( + fixture["repo_root"], + fixture["manifest_path"], + fixture["active_model"], + [dict(item) for item in REQUIRED_AUTHORITY_REQUIREMENTS], + runtime_probe=lambda: copy.deepcopy(fixture["runtime_observation"]), + ) + + +def test_governed_product_baseline_validator_accepts_evaluator_derived_fixture( + tmp_path: Path, +) -> None: + fixture = _build_governed_product_fixture(tmp_path) + + gate = _governed_baseline_gate(fixture) + + assert gate["status"] == "pass" + assert gate["violations"] == [] + assert len(gate["checked_artifacts"]) == 9 + assert all( + check["status"] == "pass" for check in gate["validation_checks"].values() + ) + assert fixture["evaluation"]["case_count"] == 70 + assert set(fixture["evaluation"]["evaluated_task_families"]) == TASKS + assert all( + metrics["case_support"] == 10 + for metrics in fixture["evaluation"]["portfolio_metrics"].values() + ) + for name in ( + "authoritative_reference_portfolio_available", + "human_review_complete", + "split_independence", + "protected_storage_isolation", + "representative_product_subgroup_support", + ): + assert gate["derived_gates"][name]["status"] == "pass" + + +@pytest.mark.parametrize( + ("role", "mutation", "expected_violation"), + [ + ( + "inference_evidence", + "cuda_unavailable", + "inference_evidence:torch_cuda_unavailable", + ), + ( + "authoritative_reference_manifest", + "authority_missing", + "authoritative_reference:missing_requirement", + ), + ( + "raw_predictions", + "prediction_missing", + "raw_predictions:", + ), + ( + "metric_report", + "subgroup_count", + "metric_report:", + ), + ( + "human_review_ledger", + "review_missing", + "human_review:", + ), + ( + "geometric_leakage_audit", + "close_pair", + "geometric_leakage:", + ), + ( + "vault_access_evidence", + "training_access", + "vault_access:", + ), + ], +) +def test_governed_product_baseline_validator_rejects_semantic_tampering( + tmp_path: Path, + role: str, + mutation: str, + expected_violation: str, +) -> None: + fixture = _build_governed_product_fixture(tmp_path) + + def mutate(payload: dict[str, Any]) -> None: + if mutation == "cuda_unavailable": + payload["torch_cuda_is_available"] = False + elif mutation == "authority_missing": + payload["entries"].pop() + payload["entries_canonical_json_sha256"] = canonical_hash( + payload["entries"] + ) + elif mutation == "prediction_missing": + missing_task = sorted(TASKS)[0] + payload["cases"] = [ + case for case in payload["cases"] if case["task"] != missing_task + ] + elif mutation == "subgroup_count": + payload["subgroups"]["dimensions"]["region"]["strata"]["flanders"][ + "case_support" + ] += 1 + payload["subgroups_canonical_json_sha256"] = canonical_hash( + payload["subgroups"] + ) + elif mutation == "review_missing": + payload["entries"].pop() + payload["entries_canonical_json_sha256"] = canonical_hash( + payload["entries"] + ) + elif mutation == "close_pair": + payload["below_threshold_pair_count"] = 1 + payload["below_threshold_pairs"] = [ + { + "development_sample_id": "development-neighbour", + "protected_sample_id": fixture["evaluation_ids"][0], + "distance_m": 1999.0, + } + ] + payload["minimum_observed_distance_m"] = 1999.0 + elif mutation == "training_access": + entry = payload["access_log"][0] + entry["purpose"] = "training" + unsigned = { + key: value for key, value in entry.items() if key != "entry_sha256" + } + entry["entry_sha256"] = canonical_hash(unsigned) + payload["access_log_canonical_json_sha256"] = canonical_hash( + payload["access_log"] + ) + else: # pragma: no cover - parametrization owns this closed set + raise AssertionError(mutation) + + _rewrite_governed_artifact(fixture, role, mutate) + gate = _governed_baseline_gate(fixture) + + assert gate["status"] == "fail" + assert any( + violation.startswith(expected_violation) for violation in gate["violations"] + ) + + +def test_governed_validator_rejects_impossible_metric_before_hash_comparison( + tmp_path: Path, +) -> None: + fixture = _build_governed_product_fixture(tmp_path) + + def mutate(payload: dict[str, Any]) -> None: + payload["results"][0]["metrics"]["precision"] = 1.5 + payload["results_canonical_json_sha256"] = canonical_hash(payload["results"]) + + _rewrite_governed_artifact(fixture, "metric_report", mutate) + gate = _governed_baseline_gate(fixture) + + assert gate["status"] == "fail" + assert any( + violation.startswith("metric_report:") + and ("range" in violation or "impossible" in violation) + for violation in gate["violations"] + ) + + +def test_governed_validator_rejects_all_empty_observation_support( + tmp_path: Path, +) -> None: + fixture = _build_governed_product_fixture(tmp_path) + + def mutate(payload: dict[str, Any]) -> None: + for metrics in payload["portfolio_metrics"].values(): + metrics["observation_support"] = { + key: 0 for key in metrics["observation_support"] + } + payload["portfolio_metrics_canonical_json_sha256"] = canonical_hash( + payload["portfolio_metrics"] + ) + + _rewrite_governed_artifact(fixture, "metric_report", mutate) + gate = _governed_baseline_gate(fixture) + + assert gate["status"] == "fail" + assert any( + violation.startswith("metric_report:") and "empty_support" in violation + for violation in gate["violations"] + ) + + +def test_governed_validator_rejects_missing_evaluator_task_family( + tmp_path: Path, +) -> None: + fixture = _build_governed_product_fixture(tmp_path) + missing_task = "terrain_interpretation" + + def mutate(payload: dict[str, Any]) -> None: + payload["cases"] = [ + case for case in payload["cases"] if case["task"] != missing_task + ] + + _rewrite_governed_artifact(fixture, "raw_predictions", mutate) + gate = _governed_baseline_gate(fixture) + + assert gate["status"] == "fail" + assert any( + violation.startswith("raw_predictions:") + and ("task" in violation or "evaluator" in violation) + for violation in gate["violations"] + ) + + +def test_governed_product_baseline_validator_rejects_manifest_claim_tampering( + tmp_path: Path, +) -> None: + fixture = _build_governed_product_fixture(tmp_path) + manifest = json.loads(fixture["manifest_path"].read_text(encoding="utf-8")) + manifest["synthetic"] = True + _write_governed_json( + fixture["repo_root"], + fixture["manifest_path"], + manifest, + ) + + gate = _governed_baseline_gate(fixture) + + assert gate["status"] == "fail" + assert "manifest:synthetic_or_unspecified" in gate["violations"] + + +def test_active_model_checksum_mismatch_is_explicit_failure(tmp_path: Path) -> None: + model_path = tmp_path / "active.pt" + model_path.write_bytes(b"observed-model") + gate = active_model_availability_gate( + { + "path": str(model_path), + "sha256": _fixture_hash("different-model"), + "size_bytes": model_path.stat().st_size, + } + ) + + assert gate["status"] == "fail" + assert gate["observed_sha256"] != gate["configured_sha256"] + assert "checksum" in gate["reason"].lower() + + +def test_missing_product_gates_fail_closed_and_explicit_fail_has_precedence() -> None: + split_result, evaluation, portfolio, firewall = evaluation_inputs() + missing_report = build_release_gate_report( + split_result, + evaluation, + portfolio, + canonical_golden_baseline(), + firewall, + {}, + ) + assert missing_report["status"] == "fail" + assert missing_report["product_benchmark_status"] == "fail" + assert set(missing_report["missing_gate_names"]["product"]) == PRODUCT_GATE_NAMES + + product_gates = {name: {"status": "pass"} for name in PRODUCT_GATE_NAMES} + product_gates["executed_product_incumbent_baseline"] = {"status": "not_evaluable"} + product_gates["human_review_complete"] = {"status": "fail"} + precedence_report = build_release_gate_report( + split_result, + evaluation, + portfolio, + canonical_golden_baseline(), + firewall, + product_gates, + ) + assert precedence_report["status"] == "fail" + assert precedence_report["product_benchmark_status"] == "fail" + + +def test_document_status_booleans_cannot_spoof_review_split_or_vault( + tmp_path: Path, +) -> None: + fixture = _build_governed_product_fixture(tmp_path) + snapshot = { + "active_model": fixture["active_model"], + "authority_requirements": [ + dict(item) for item in REQUIRED_AUTHORITY_REQUIREMENTS + ], + "v56_review_and_split": { + "review_complete": True, + "reviewed_sample_count": 999, + "sample_count": 999, + "split_independence_proven": True, + "cross_split_pairs_below_2000_m": 0, + }, + "protected_test_isolation": True, + "phase3_leakage_status": "pass", + } + missing_manifest = ( + fixture["repo_root"] + / "artifacts/evidence/accuracy/P4/missing/product-baseline-manifest.json" + ) + + gates = product_gate_evidence(fixture["repo_root"], snapshot, missing_manifest) + + for name in ( + "human_review_complete", + "split_independence", + "protected_storage_isolation", + ): + assert gates[name]["status"] in {"fail", "not_evaluable"} + assert gates[name]["status"] != "pass" + + +def test_all_mandatory_product_gates_make_phase5_reachable() -> None: + split_result, evaluation, portfolio, firewall = evaluation_inputs() + product_gates = {name: {"status": "pass"} for name in PRODUCT_GATE_NAMES} + + report = build_release_gate_report( + split_result, + evaluation, + portfolio, + canonical_golden_baseline(), + firewall, + product_gates, + ) + + assert report["status"] == "pass" + assert report["phase_decision"] == "ready_for_phase5" + assert report["missing_gate_names"] == {"local": [], "product": []} + assert set( + report["local_gates"]["normative_split_roles_and_leakage"]["required_roles"] + ) == {"train", "val", "calibration", "test", "background-test", "challenge"} + + def test_split_fixture_is_order_independent_and_has_all_roles() -> None: source = load_source() - development, protected, leakage = build_manifests(source) + development, protected, leakage = build_manifests(source, trusted_fixture_mode=True) reversed_source = copy.deepcopy(source) reversed_source["samples"].reverse() reversed_development, reversed_protected, reversed_leakage = build_manifests( - reversed_source + reversed_source, trusted_fixture_mode=True ) assert leakage["status"] == "pass" @@ -81,20 +1018,26 @@ def test_split_fixture_is_order_independent_and_has_all_roles() -> None: def test_training_firewall_rejects_non_train_and_protected_lineage() -> None: - development, protected, leakage = build_manifests(load_source()) + development, protected, leakage = build_manifests( + load_source(), trusted_fixture_mode=True + ) assert leakage["status"] == "pass" train = [item for item in development["samples"] if item["split"] == "train"] validation = next(item for item in development["samples"] if item["split"] == "val") - assert_training_inputs_safe([], train, protected) + assert_training_inputs_safe([], train, protected, trusted_fixture_mode=True) with pytest.raises(LeakageError, match="non_train_role"): - assert_training_inputs_safe([], [validation], protected) + assert_training_inputs_safe( + [], [validation], protected, trusted_fixture_mode=True + ) with pytest.raises(LeakageError, match="protected_identity"): disguised = dict(train[0]) disguised["source_family"] = protected["samples"][0]["source_family"] - assert_training_inputs_safe([], [disguised], protected) + assert_training_inputs_safe( + [], [disguised], protected, trusted_fixture_mode=True + ) with pytest.raises(LeakageError, match="protected_path"): - assert_training_inputs_safe([CASES], [], protected) + assert_training_inputs_safe([CASES], [], protected, trusted_fixture_mode=True) def test_task_evaluator_retains_exact_raw_inputs_metrics_and_failures() -> None: @@ -103,7 +1046,7 @@ def test_task_evaluator_retains_exact_raw_inputs_metrics_and_failures() -> None: assert report["task_count"] == 7 assert report["case_count"] == 9 assert len(report["task_inventory"]) >= 15 - assert len(report["failures"]) == 11 + assert len(report["failures"]) >= 11 assert report["subgroups"]["overall_status"] == "not_evaluable" assert all( { @@ -179,8 +1122,8 @@ def test_one_workflow_is_byte_reproducible_complete_and_fail_closed( assert first == second assert first_bytes == second_bytes assert first["local_harness_status"] == "pass" - assert first["product_benchmark_status"] == "not_evaluable" - assert first["status"] == "not_evaluable" + assert first["product_benchmark_status"] == "fail" + assert first["status"] == "fail" assert first["phase4_done"] is False assert first["phase5_ready"] is False required = { @@ -230,6 +1173,21 @@ def test_one_workflow_is_byte_reproducible_complete_and_fail_closed( "not_evaluable", } + input_manifest = json.loads( + (output / "input-manifest.json").read_text(encoding="utf-8") + ) + benchmark_manifest = json.loads( + (output / "benchmark-manifest.json").read_text(encoding="utf-8") + ) + assert input_manifest["product_baseline"]["validation_status"] == "not_evaluable" + assert input_manifest["product_baseline"]["artifacts"] == [] + assert benchmark_manifest["product_baseline"] == input_manifest["product_baseline"] + assert ( + benchmark_manifest["product_gate_evidence_sha256"] + == first["product_gate_evidence_sha256"] + ) + assert first["evidence_run_id"].startswith("p4-2.0.0-") + def test_immutable_workflow_refuses_to_replace_changed_evidence(tmp_path: Path) -> None: output = tmp_path / "p4" @@ -238,3 +1196,14 @@ def test_immutable_workflow_refuses_to_replace_changed_evidence(tmp_path: Path) with pytest.raises(EvidenceConflictError, match="Refusing to overwrite"): run_workflow(ROOT, output) + + +def test_immutable_workflow_rejects_rogue_nested_evidence(tmp_path: Path) -> None: + output = tmp_path / "p4" + run_workflow(ROOT, output) + rogue = output / "rogue" / "unmanifested.json" + rogue.parent.mkdir() + rogue.write_text("{}\n", encoding="utf-8") + + with pytest.raises(EvidenceConflictError, match="(?i)unexpected|immutable"): + run_workflow(ROOT, output) diff --git a/backend/tests/test_accuracy_phase4_evaluator_hardening.py b/backend/tests/test_accuracy_phase4_evaluator_hardening.py index 7cedbe4b..a7528a66 100644 --- a/backend/tests/test_accuracy_phase4_evaluator_hardening.py +++ b/backend/tests/test_accuracy_phase4_evaluator_hardening.py @@ -17,6 +17,7 @@ if str(SCRIPTS) not in sys.path: from accuracy_phase4_evaluator import ( # noqa: E402 TASKS, + EXPECTED_PROTECTED_POLICY, canonical_hash, count_metrics, detection_ap, @@ -40,6 +41,7 @@ METADATA = { "source": "synthetic-source", "sensor": "synthetic-sensor", "resolution_m": 0.25, + "context": "dense_urban", "season": "summer", "date": "2026-01-01", "vegetation": "partial", @@ -149,6 +151,7 @@ def test_raw_evidence_and_hashes_are_exact_and_recomputable(tmp_path: Path) -> N case = detection_case() portfolio = { "schema_version": 2, + "portfolio_kind": "synthetic_contract", "portfolio_id": "synthetic-hardening-test", "portfolio_lineage": { "origin": "repository_fixture", @@ -158,7 +161,7 @@ def test_raw_evidence_and_hashes_are_exact_and_recomputable(tmp_path: Path) -> N "split_roles": ["test"], "selection_policy": "Fixed before evaluation; no selection.", "claim_boundary": "Synthetic evaluator test; not product accuracy.", - "protected_policy": {"threshold_selection_allowed": False}, + "protected_policy": copy.deepcopy(EXPECTED_PROTECTED_POLICY), "cases": [case], } path = tmp_path / "portfolio.json" @@ -203,6 +206,105 @@ def test_raw_evidence_and_hashes_are_exact_and_recomputable(tmp_path: Path) -> N evaluate_cases(path, {case["sample_id"]}) +def test_portfolio_schema_policy_metadata_and_lineage_are_strict( + tmp_path: Path, +) -> None: + case = detection_case("strict-contract") + portfolio = { + "schema_version": 2, + "portfolio_kind": "synthetic_contract", + "portfolio_id": "synthetic-strict-contract", + "portfolio_lineage": { + "origin": "repository_fixture", + "source_path": "synthetic.json", + "version": "1", + }, + "split_roles": ["test"], + "selection_policy": "Fixed before evaluation; no selection.", + "claim_boundary": "Synthetic evaluator test; not product accuracy.", + "protected_policy": copy.deepcopy(EXPECTED_PROTECTED_POLICY), + "cases": [case], + } + path = tmp_path / "strict.json" + + def evaluate(value: dict) -> dict: + path.write_text(json.dumps(value, ensure_ascii=False), encoding="utf-8") + return evaluate_cases(path, {case["sample_id"]}) + + assert evaluate(portfolio)["case_count"] == 1 + + for invalid_version in (1, True, "2"): + invalid = copy.deepcopy(portfolio) + invalid["schema_version"] = invalid_version + with pytest.raises( + ValueError, match="schema_version must be exactly integer 2" + ): + evaluate(invalid) + + invalid_policy = copy.deepcopy(portfolio) + invalid_policy["protected_policy"]["test_feedback_allowed"] = True + with pytest.raises(ValueError, match="protected_policy must exactly equal"): + evaluate(invalid_policy) + + invalid_metadata = copy.deepcopy(portfolio) + invalid_metadata["cases"][0]["metadata"]["source"] = "unknown" + with pytest.raises(ValueError, match="metadata.source must be a meaningful"): + evaluate(invalid_metadata) + + invalid_resolution = copy.deepcopy(portfolio) + invalid_resolution["cases"][0]["metadata"]["resolution_m"] = 0 + with pytest.raises(ValueError, match="metadata.resolution_m must be positive"): + evaluate(invalid_resolution) + + invalid_lineage = copy.deepcopy(portfolio) + del invalid_lineage["cases"][0]["lineage"]["prediction"]["derivation"] + with pytest.raises(ValueError, match="lineage.prediction missing"): + evaluate(invalid_lineage) + + +def test_portfolio_kind_separates_synthetic_and_governed_product_claims( + tmp_path: Path, +) -> None: + fixture_path = ( + ROOT / "fixtures" / "accuracy" / "p4" / "protected-baseline-cases.json" + ) + synthetic = json.loads(fixture_path.read_text(encoding="utf-8")) + allowed = {item["sample_id"] for item in synthetic["cases"]} + path = tmp_path / "portfolio.json" + + missing_kind = copy.deepcopy(synthetic) + del missing_kind["portfolio_kind"] + path.write_text(json.dumps(missing_kind), encoding="utf-8") + with pytest.raises(ValueError, match="portfolio_kind"): + evaluate_cases(path, allowed) + + confused = copy.deepcopy(synthetic) + confused["claim_boundary"] = "Governed product baseline accuracy evidence." + path.write_text(json.dumps(confused), encoding="utf-8") + with pytest.raises(ValueError, match="synthetic_contract"): + evaluate_cases(path, allowed) + + governed = json.loads( + json.dumps(synthetic) + .replace("Synthetic", "Governed") + .replace("synthetic", "governed") + .replace("repository_fixture", "governed_product_evaluation") + ) + governed["portfolio_kind"] = "governed_product_baseline" + governed["portfolio_id"] = "governed-product-baseline-test" + governed["claim_boundary"] = ( + "Governed product baseline metrics recomputed from protected raw cases; " + "inference provenance is validated separately." + ) + path.write_text(json.dumps(governed), encoding="utf-8") + report = evaluate_cases(path, allowed) + assert report["portfolio_kind"] == "governed_product_baseline" + assert set(report["evaluated_task_families"]) == TASKS + + governed["cases"] = governed["cases"][:-1] + path.write_text(json.dumps(governed), encoding="utf-8") + + def test_ap_ties_use_stable_ids_and_matching_is_class_aware() -> None: references = [{"id": "r", "class": "building", "bbox": [0, 0, 4, 4]}] predictions = [ @@ -233,6 +335,83 @@ def test_ap_ties_use_stable_ids_and_matching_is_class_aware() -> None: assert detection_ap(wrong_class, references, 0.5) == pytest.approx(0.5) +def test_detection_ap_and_calibration_are_pooled_globally_and_per_subgroup( + tmp_path: Path, +) -> None: + first = detection_case("a-case") + first["predictions"] = [ + { + "id": "p-true", + "class": "building", + "bbox": [0, 0, 4, 4], + "confidence": 0.9, + } + ] + second = detection_case("b-case") + second["predictions"] = [ + { + "id": "p-false", + "class": "building", + "bbox": [10, 10, 12, 12], + "confidence": 0.9, + }, + { + "id": "p-true", + "class": "building", + "bbox": [0, 0, 4, 4], + "confidence": 0.8, + }, + ] + portfolio = { + "schema_version": 2, + "portfolio_kind": "synthetic_contract", + "portfolio_id": "synthetic-pooled-detection", + "portfolio_lineage": { + "origin": "repository_fixture", + "source_path": "pooled.json", + "version": "1", + }, + "split_roles": ["test"], + "selection_policy": "Fixed before evaluation; no selection.", + "claim_boundary": "Synthetic evaluator test; not product accuracy.", + "protected_policy": copy.deepcopy(EXPECTED_PROTECTED_POLICY), + "cases": [first, second], + } + path = tmp_path / "pooled.json" + path.write_text(json.dumps(portfolio, ensure_ascii=False), encoding="utf-8") + report = evaluate_cases(path, {"a-case", "b-case"}) + + case_ap = [item["metrics"]["ap50"] for item in report["results"]] + pooled = report["portfolio_metrics"]["object_detection"]["micro"] + expected_pooled = detection_ap( + [ + {**item, "id": f"a-case::{item['id']}", "_sample_id": "a-case"} + for item in first["predictions"] + ] + + [ + {**item, "id": f"b-case::{item['id']}", "_sample_id": "b-case"} + for item in second["predictions"] + ], + [ + {**item, "id": f"a-case::{item['id']}", "_sample_id": "a-case"} + for item in first["references"] + ] + + [ + {**item, "id": f"b-case::{item['id']}", "_sample_id": "b-case"} + for item in second["references"] + ], + 0.5, + ) + assert pooled["ap50"] == expected_pooled + assert pooled["ap50"] != pytest.approx(sum(case_ap) / len(case_ap)) + assert sum(item["count"] for item in pooled["calibration"]["bins"]) == 3 + subgroup = report["subgroups"]["dimensions"]["region"]["strata"]["flanders"] + subgroup_calibration = subgroup["task_metrics"]["object_detection"]["micro"][ + "calibration" + ] + assert sum(item["count"] for item in subgroup_calibration["bins"]) == 3 + + def test_raster_requires_exact_rectangular_alignment_masks_nodata_and_classes() -> None: invalid_case = detection_case("invalid-class") invalid_case["predictions"][0]["class"] = "road" @@ -270,6 +449,11 @@ def test_raster_requires_exact_rectangular_alignment_masks_nodata_and_classes() invalid_nodata["predictions"][0][0] = -9999 with pytest.raises(ValueError, match="marks nodata as valid"): evaluate_raster_classification(invalid_nodata) + singular = raster_case("singular") + for side in ("reference", "prediction"): + singular["raster_context"][side]["transform"] = [1, 2, 0, 2, 4, 0] + with pytest.raises(ValueError, match="affine transform is singular"): + evaluate_raster_classification(singular) def test_polygon_metrics_require_valid_geometry_projected_crs_and_metres() -> None: @@ -280,11 +464,59 @@ def test_polygon_metrics_require_valid_geometry_projected_crs_and_metres() -> No ]["mean_iou"] == 1.0 ) + outer = [ + [100000, 200000], + [100020, 200000], + [100020, 200020], + [100000, 200020], + [100000, 200000], + ] + hole = [ + [100005, 200005], + [100010, 200005], + [100010, 200010], + [100005, 200010], + [100005, 200005], + ] + polygon_geometry = {"type": "Polygon", "coordinates": [outer, hole]} + geojson_polygon = polygon_case() + for side in ("references", "predictions"): + del geojson_polygon[side][0]["polygon"] + geojson_polygon[side][0]["geometry"] = copy.deepcopy(polygon_geometry) + polygon_result = evaluate_vector_comparison(geojson_polygon) + assert polygon_result["metrics"]["mean_iou"] == 1.0 + assert polygon_result["raw"]["references"][0]["geometry"] == polygon_geometry + + second = [ + [100030, 200000], + [100040, 200000], + [100040, 200010], + [100030, 200010], + [100030, 200000], + ] + multipolygon_geometry = { + "type": "MultiPolygon", + "coordinates": [[outer, hole], [second]], + } + geojson_multi = polygon_case() + for side in ("references", "predictions"): + del geojson_multi[side][0]["polygon"] + geojson_multi[side][0]["geometry"] = copy.deepcopy(multipolygon_geometry) + assert evaluate_vector_comparison(geojson_multi)["metrics"]["mean_iou"] == 1.0 geographic = polygon_case() geographic["spatial_context"]["crs"] = "EPSG:4326" with pytest.raises(ValueError, match="projected CRS"): evaluate_vector_comparison(geographic) + mercator = polygon_case() + mercator["spatial_context"]["crs"] = "EPSG:3857" + with pytest.raises(ValueError, match="Mercator is unsuitable"): + evaluate_vector_comparison(mercator) + + wrong_geography = polygon_case() + wrong_geography["spatial_context"]["crs"] = "EPSG:32660" + with pytest.raises(ValueError, match="does not overlap"): + evaluate_vector_comparison(wrong_geography) wrong_units = polygon_case() wrong_units["spatial_context"]["coordinate_units"] = "degree" @@ -303,6 +535,61 @@ def test_polygon_metrics_require_valid_geometry_projected_crs_and_metres() -> No evaluate_vector_comparison(bowtie) +def test_failure_gallery_covers_geometry_raster_calibration_and_contexts() -> None: + segmentation = polygon_case("footprint_segmentation") + segmentation["predictions"][0]["polygon"] = [ + [100000, 200000], + [100012, 200000], + [100012, 200010], + [100000, 200010], + [100000, 200000], + ] + segmentation_result = evaluate_footprint_segmentation(segmentation) + segmentation_codes = { + item["error_code"] for item in segmentation_result["failures"] + } + assert {"M-BOUNDARY", "M-AREA-BIAS"} <= segmentation_codes + + raster = raster_case("raster-taxonomy") + raster["metadata"]["tile_edge"] = True + raster["predictions"][0][0] = 1 + raster_result = evaluate_raster_classification(raster) + raster_failure = next( + item + for item in raster_result["failures"] + if item["kind"] == "raster_misclassification" + ) + assert raster_failure["error_code"] == "M-CLASS" + assert "tile_edge" in raster_failure["contexts"] + + detection = detection_case("context-taxonomy") + detection["references"] = [] + detection["predictions"] = [ + { + "id": "high-confidence-fp", + "class": "building", + "bbox": [10, 10, 12, 12], + "confidence": 0.95, + } + ] + detection["config"]["fixed_diagnostic_risk_thresholds"] = [0.5, 0.9] + detection["metadata"]["tile_edge"] = True + detection["metadata"]["ood"] = True + detection_result = evaluate_object_detection(detection) + false_positive = next( + item + for item in detection_result["failures"] + if item["kind"] == "false_positive" + ) + assert {"tile_edge", "high_confidence", "out_of_distribution"} <= set( + false_positive["contexts"] + ) + assert {"M-MISCALIBRATED", "M-OOD"} <= set(false_positive["secondary_error_codes"]) + assert any( + item["error_code"] == "M-MISCALIBRATED" for item in detection_result["failures"] + ) + + def test_terrain_rejects_non_finite_and_validation_counts_only_critical_misses() -> ( None ): @@ -345,26 +632,60 @@ def test_terrain_rejects_non_finite_and_validation_counts_only_critical_misses() evaluate_validation(validation)["metrics"]["blocker_or_critical_miss_count"] == 1 ) + validation["expected_anomalies"] = [{"code": "D-SEVERITY", "severity": "critical"}] + validation["observed_anomalies"] = [{"code": "D-SEVERITY", "severity": "minor"}] + severity_result = evaluate_validation(validation) + assert severity_result["metrics"]["true_positive"] == 0 + assert severity_result["metrics"]["false_positive"] == 1 + assert severity_result["metrics"]["false_negative"] == 1 + assert severity_result["metrics"]["severity_mismatch_count"] == 1 + assert severity_result["metrics"]["blocker_or_critical_miss_count"] == 1 + validation["expected_anomalies"] = ["D-NO-SEVERITY"] with pytest.raises(ValueError, match="include code and severity"): evaluate_validation(validation) -def _subgroup_result(region: str, tp: int, fp: int, fn: int) -> dict: +def _subgroup_result(region: str, index: int, tp: int, fp: int, fn: int) -> dict: metadata = copy.deepcopy(METADATA) metadata["region"] = region + sample_id = f"{region}-{index}" + reference = {"id": "r", "class": "building", "bbox": [0, 0, 1, 1]} + prediction = { + "id": "p", + "class": "building", + "bbox": [0, 0, 1, 1], + "confidence": 0.8, + } return { + "sample_id": sample_id, "task": "object_detection", "metadata": metadata, "metrics": {**count_metrics(tp, fp, fn), "ap50": 0.5, "ap50_95": 0.4}, + "raw": { + "sample_id": sample_id, + "classes": ["building"], + "references": [reference], + "predictions_pre_filter": [prediction], + "predictions_post_filter": [prediction], + "matches": [ + { + "prediction_id": "p", + "reference_id": "r", + "overlap": 1.0, + "confidence": 0.8, + "class": "building", + } + ], + }, "failures": [], } def test_subgroups_report_task_metrics_support_ci_and_worst_stratum() -> None: results = [ - *[_subgroup_result("strong", 10, 0, 0) for _ in range(5)], - *[_subgroup_result("weak", 1, 4, 4) for _ in range(5)], + *[_subgroup_result("strong", index, 10, 0, 0) for index in range(5)], + *[_subgroup_result("weak", index, 1, 4, 4) for index in range(5)], ] report = subgroup_report(results) region = report["dimensions"]["region"] @@ -376,7 +697,7 @@ def test_subgroups_report_task_metrics_support_ci_and_worst_stratum() -> None: assert weak["macro"]["f1_case_support"] == 5 assert region["worst_stratum_by_task"]["object_detection"]["stratum"] == "weak" - insufficient = subgroup_report([_subgroup_result("thin", 1, 0, 0)]) + insufficient = subgroup_report([_subgroup_result("thin", 0, 1, 0, 0)]) thin = insufficient["dimensions"]["region"]["strata"]["thin"] assert thin["task_metrics"]["object_detection"]["status"] == "insufficient_support" assert thin["release_gate_status"] == "not_evaluable" diff --git a/backend/tests/test_accuracy_phase4_split_hardening.py b/backend/tests/test_accuracy_phase4_split_hardening.py index a0caf1fc..dd8db3e2 100644 --- a/backend/tests/test_accuracy_phase4_split_hardening.py +++ b/backend/tests/test_accuracy_phase4_split_hardening.py @@ -1,6 +1,7 @@ from __future__ import annotations import copy +import hashlib import json import sys from pathlib import Path @@ -28,13 +29,25 @@ def load_source() -> dict: return json.loads(SOURCE.read_text(encoding="utf-8")) +def build_fixture_manifests(source: dict) -> tuple[dict, dict, dict]: + return build_manifests(source, trusted_fixture_mode=True) + + +def assert_fixture_training_inputs_safe( + input_paths: list[Path], input_records: list[dict], protected: dict +) -> None: + assert_training_inputs_safe( + input_paths, input_records, protected, trusted_fixture_mode=True + ) + + def test_normative_roles_hashes_and_source_order_are_enforced() -> None: source = load_source() - development, protected, leakage = build_manifests(source) + development, protected, leakage = build_fixture_manifests(source) reversed_source = copy.deepcopy(source) reversed_source["samples"].reverse() - reversed_development, reversed_protected, reversed_leakage = build_manifests( - reversed_source + reversed_development, reversed_protected, reversed_leakage = ( + build_fixture_manifests(reversed_source) ) assert leakage["status"] == "pass" @@ -82,7 +95,7 @@ def test_cross_split_lineage_and_content_collisions_fail( ) -> None: source = load_source() source["samples"][8][field] = source["samples"][0][field] - _development, _protected, leakage = build_manifests(source) + _development, _protected, leakage = build_fixture_manifests(source) assert leakage["status"] == "fail" assert expected_code in {item["code"] for item in leakage["findings"]} @@ -98,7 +111,7 @@ def test_cross_split_lineage_and_content_collisions_fail( def test_near_duplicate_fingerprints_fail(field: str, expected_code: str) -> None: source = load_source() source["samples"][8][field] = source["samples"][0][field] - _development, _protected, leakage = build_manifests(source) + _development, _protected, leakage = build_fixture_manifests(source) assert leakage["status"] == "fail" assert expected_code in {item["code"] for item in leakage["findings"]} @@ -111,7 +124,7 @@ def test_object_native_feature_and_spatial_collisions_fail() -> None: "native_feature_ids" ] source["samples"][10]["bbox"] = source["samples"][2]["bbox"] - _development, _protected, leakage = build_manifests(source) + _development, _protected, leakage = build_fixture_manifests(source) codes = {item["code"] for item in leakage["findings"]} assert {"S-OBJECT-INSTANCE", "S-NATIVE-FEATURE", "S-SPATIAL-OVERLAP"} <= codes @@ -121,32 +134,34 @@ def test_non_metric_crs_and_missing_normative_role_fail_closed() -> None: geographic = load_source() geographic["crs"] = "EPSG:4326" with pytest.raises(LeakageError, match="projected in metres"): - build_manifests(geographic) + build_fixture_manifests(geographic) missing = load_source() missing["samples"] = [ item for item in missing["samples"] if item["split"] != "calibration" ] with pytest.raises(LeakageError, match="Required splits are absent"): - build_manifests(missing) + build_fixture_manifests(missing) def test_training_firewall_only_allows_train_and_binds_protected_lineage() -> None: - development, protected, leakage = build_manifests(load_source()) + development, protected, leakage = build_fixture_manifests(load_source()) assert leakage["status"] == "pass" train = [item for item in development["samples"] if item["split"] == "train"] validation = next(item for item in development["samples"] if item["split"] == "val") protected_item = protected["samples"][0] - assert_training_inputs_safe([], train, protected) + assert_fixture_training_inputs_safe([], train, protected) with pytest.raises(LeakageError, match="non_train_role"): - assert_training_inputs_safe([], [validation], protected) + assert_fixture_training_inputs_safe([], [validation], protected) with pytest.raises(LeakageError, match="protected_identity"): disguised = copy.deepcopy(train[0]) disguised["source_family"] = protected_item["source_family"] - assert_training_inputs_safe([], [disguised], protected) + assert_fixture_training_inputs_safe([], [disguised], protected) with pytest.raises(LeakageError, match="protected_path"): - assert_training_inputs_safe([Path("vault/protected/test.json")], [], protected) + assert_fixture_training_inputs_safe( + [Path("vault/protected/test.json")], [], protected + ) def test_failed_generation_writes_status_but_no_consumable_manifests( @@ -159,12 +174,18 @@ def test_failed_generation_writes_status_but_no_consumable_manifests( output = tmp_path / "out" with pytest.raises(LeakageError, match="Leakage gate failed"): - generate(source_path, output) + generate(source_path, output, trusted_fixture_mode=True) status = json.loads((output / "generation-status.json").read_text(encoding="utf-8")) assert status["status"] == "fail" - assert not (output / "development-split-manifest.json").exists() - assert not (output / "protected-split-manifest.json").exists() + assert status["consumable_manifests_valid"] is False + for name in ( + "development-split-manifest.json", + "protected-split-manifest.json", + ): + tombstone = json.loads((output / name).read_text(encoding="utf-8")) + assert tombstone["status"] == "invalidated" + assert tombstone["consumable"] is False def test_deterministic_grouped_assignment_is_stable_and_keeps_relatives_together() -> ( @@ -196,11 +217,11 @@ def test_deterministic_grouped_assignment_is_stable_and_keeps_relatives_together item.pop("split") source["samples"][1]["group_id"] = source["samples"][0]["group_id"] - development, protected, leakage = build_manifests(source) + development, protected, leakage = build_fixture_manifests(source) reversed_source = copy.deepcopy(source) reversed_source["samples"].reverse() - reversed_development, reversed_protected, reversed_leakage = build_manifests( - reversed_source + reversed_development, reversed_protected, reversed_leakage = ( + build_fixture_manifests(reversed_source) ) assigned = { @@ -220,3 +241,298 @@ def test_deterministic_grouped_assignment_is_stable_and_keeps_relatives_together assert reversed_development["manifest_sha256"] == development["manifest_sha256"] assert reversed_protected["manifest_sha256"] == protected["manifest_sha256"] assert reversed_leakage == leakage + + +def test_source_cannot_weaken_mandatory_roles_or_policy_floors() -> None: + source = load_source() + source["required_splits"] = ["train", "val", "test"] + with pytest.raises(LeakageError, match="mandatory role order"): + build_fixture_manifests(source) + + grouped = load_source() + grouped["assignment_mode"] = "deterministic_grouped" + grouped["split_assignment"] = {"roles": ["train", "val"]} + for item in grouped["samples"]: + item.pop("split") + with pytest.raises(LeakageError, match="mandatory role order"): + build_fixture_manifests(grouped) + + for field, value in ( + ("independence_buffer_m", 1999), + ("perceptual_hamming_threshold", 3), + ("label_geometry_hamming_threshold", 1), + ): + weakened = load_source() + weakened[field] = value + with pytest.raises(LeakageError, match="code-owned minimum"): + build_fixture_manifests(weakened) + + +def test_task_coverage_gap_and_even_justified_exemption_fail_honestly() -> None: + source = load_source() + source["samples"] = [ + item + for item in source["samples"] + if item["sample_id"] != "validation-test-national" + ] + _development, _protected, leakage = build_fixture_manifests(source) + assert leakage["status"] == "fail" + assert "S-PROTECTED-TASK-COVERAGE-MISSING" in { + finding["code"] for finding in leakage["findings"] + } + + source["protected_task_exemptions"] = { + "geospatial_data_validation": ( + "No evaluator-visible reference exists; challenge data remains sealed." + ) + } + _development, _protected, leakage = build_fixture_manifests(source) + assert leakage["status"] == "fail" + assert "S-PROTECTED-TASK-COVERAGE-EXEMPTED" in { + finding["code"] for finding in leakage["findings"] + } + + +def test_identifiers_and_acquisition_dates_are_canonical_leakage_keys() -> None: + source = load_source() + source["samples"][8]["group_id"] = " G01 " + _development, _protected, leakage = build_fixture_manifests(source) + assert "S-SPATIAL-GROUP" in {item["code"] for item in leakage["findings"]} + + temporal = load_source() + temporal["samples"][8]["acquisition_date"] = temporal["samples"][0][ + "acquisition_date" + ] + _development, _protected, leakage = build_fixture_manifests(temporal) + assert "S-ACQUISITION-DATE" in {item["code"] for item in leakage["findings"]} + + ambiguous = load_source() + ambiguous["samples"][8]["sample_id"] = " DET-TRAIN-A " + with pytest.raises(LeakageError, match="ambiguous canonical sample_id"): + build_fixture_manifests(ambiguous) + + +def test_challenge_is_sealed_in_standard_manifest() -> None: + _development, protected, leakage = build_fixture_manifests(load_source()) + assert leakage["status"] == "pass" + challenge = [item for item in protected["samples"] if item["split"] == "challenge"] + forbidden = { + "label_sha256", + "label_geometry_hash", + "label_geometry_fingerprint", + "object_ids", + "native_feature_ids", + "record_sha256", + "label_path", + "label_geometry_path", + } + assert challenge + assert all(item["sealed"] is True for item in challenge) + assert all(not (forbidden & set(item)) for item in challenge) + + +def test_firewall_rejects_empty_tampered_wrong_and_renamed_manifests( + tmp_path: Path, +) -> None: + development, protected, leakage = build_fixture_manifests(load_source()) + assert leakage["status"] == "pass" + train = [item for item in development["samples"] if item["split"] == "train"] + + with pytest.raises(LeakageError, match="empty manifest"): + assert_fixture_training_inputs_safe([], train, {}) + with pytest.raises(LeakageError, match="missing fields"): + assert_fixture_training_inputs_safe([], train, development) + tampered = copy.deepcopy(protected) + tampered["samples"].pop() + with pytest.raises(LeakageError, match="checksum mismatch"): + assert_fixture_training_inputs_safe([], train, tampered) + + renamed = tmp_path / "ordinary-training-input.json" + renamed.write_text(json.dumps(protected), encoding="utf-8") + with pytest.raises(LeakageError, match="protected_manifest_content"): + assert_fixture_training_inputs_safe([renamed], train, protected) + + +def _canonical_json_sha256(value: object) -> str: + return hashlib.sha256( + json.dumps( + value, ensure_ascii=False, sort_keys=True, separators=(",", ":") + ).encode("utf-8") + ).hexdigest() + + +def make_governed_source(tmp_path: Path) -> dict: + source = load_source() + source["dataset_version"] = "governed-production-v1" + source["claim_boundary"] = "Governed production split source." + p3_items: list[dict] = [] + provenance_records: list[dict] = [] + asset_fields = { + "raw_image": ("raw_image_path", "raw_image_sha256"), + "processed_image": ("processed_image_path", "processed_image_sha256"), + "label": ("label_path", "label_sha256"), + "label_geometry": ("label_geometry_path", "label_geometry_hash"), + } + for sample in source["samples"]: + assets: dict[str, dict] = {} + p3_ids: dict[str, str] = {} + for role, (path_field, hash_field) in asset_fields.items(): + relative = Path("assets") / sample["sample_id"] / f"{role}.bin" + path = tmp_path / relative + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(f"{sample['sample_id']}:{role}:governed".encode()) + digest = hashlib.sha256(path.read_bytes()).hexdigest() + p3_id = hashlib.sha256( + f"{sample['sample_id']}:{role}".encode() + ).hexdigest()[:20] + relative_posix = relative.as_posix() + sample[path_field] = relative_posix + sample[hash_field] = digest + p3_ids[role] = p3_id + assets[role] = { + "path": relative_posix, + "sha256": digest, + "size_bytes": path.stat().st_size, + "p3_item_id": p3_id, + } + p3_items.append( + { + "item_id": p3_id, + "path": relative_posix, + "sha256": digest, + "size_bytes": path.stat().st_size, + "status": "examined", + "read_status": "readable", + "recommended_action": "accept", + "empty_content": False, + "schema_conformity": "conformant", + "anomalies": [], + } + ) + provenance_id = ( + "prov-" + hashlib.sha256(sample["sample_id"].encode()).hexdigest()[:24] + ) + sample["governance_binding"] = { + "source_provenance_record_id": provenance_id, + "p3_item_ids": p3_ids, + } + provenance_records.append( + { + "record_id": provenance_id, + "sample_id": sample["sample_id"], + "status": "accepted", + "lineage_status": "complete", + "training_allowed": True, + "perceptual_image_hash": sample["perceptual_image_hash"], + "label_geometry_fingerprint": sample["label_geometry_fingerprint"], + "assets": assets, + } + ) + p3 = { + "schema_version": 1, + "scan_id": "p3-test-governed", + "scanner_version": "3.0.3", + "completed_at": "2026-08-02T12:00:00+02:00", + "items": p3_items, + "reconciliation": { + "examined": len(p3_items), + "skipped": 0, + "unreachable": 0, + "inventory_total": len(p3_items), + "reconciles": True, + }, + } + provenance = { + "schema_version": 1, + "manifest_type": "geointel_phase4_source_provenance", + "status": "pass", + "records": provenance_records, + "records_canonical_json_sha256": _canonical_json_sha256(provenance_records), + } + p3_path = tmp_path / "p3.json" + provenance_path = tmp_path / "provenance.json" + p3_path.write_text(json.dumps(p3), encoding="utf-8") + provenance_path.write_text(json.dumps(provenance), encoding="utf-8") + source["governance_evidence"] = { + "p3_scan_manifest": { + "path": p3_path.name, + "sha256": hashlib.sha256(p3_path.read_bytes()).hexdigest(), + }, + "source_provenance_manifest": { + "path": provenance_path.name, + "sha256": hashlib.sha256(provenance_path.read_bytes()).hexdigest(), + }, + } + return source + + +def test_fixture_mode_is_explicit_and_source_metadata_cannot_enable_it() -> None: + with pytest.raises(LeakageError, match="explicit trusted_fixture_mode"): + build_manifests(load_source()) + + source = load_source() + source["trusted_fixture_mode"] = True + with pytest.raises(LeakageError, match="cannot be enabled by source metadata"): + build_manifests(source, trusted_fixture_mode=True) + + +def test_empty_or_arbitrary_governance_json_is_rejected(tmp_path: Path) -> None: + source = load_source() + source["dataset_version"] = "governed-production-v1" + source["claim_boundary"] = "Governed production split source." + p3 = tmp_path / "p3.json" + provenance = tmp_path / "provenance.json" + p3.write_text("{}", encoding="utf-8") + provenance.write_text("{}", encoding="utf-8") + source["governance_evidence"] = { + "p3_scan_manifest": { + "path": p3.name, + "sha256": hashlib.sha256(p3.read_bytes()).hexdigest(), + }, + "source_provenance_manifest": { + "path": provenance.name, + "sha256": hashlib.sha256(provenance.read_bytes()).hexdigest(), + }, + } + with pytest.raises(LeakageError, match="non-empty JSON object"): + build_manifests(source, source_root=tmp_path) + + +def test_governed_records_require_exact_provenance_paths_and_live_bytes( + tmp_path: Path, +) -> None: + source = make_governed_source(tmp_path) + development, protected, leakage = build_manifests(source, source_root=tmp_path) + assert leakage["status"] == "pass" + assert protected["source_trust"]["production_accuracy_use_allowed"] is True + train = [item for item in development["samples"] if item["split"] == "train"] + assert_training_inputs_safe([], train, protected) + + no_paths = copy.deepcopy(train[0]) + no_paths.pop("content_path_bindings") + with pytest.raises(LeakageError, match="missing_accessible_content_paths"): + assert_training_inputs_safe([], [no_paths], protected) + + relabeled = copy.deepcopy( + next(item for item in protected["samples"] if item["split"] == "test") + ) + relabeled["split"] = "train" + relabeled.pop("content_path_bindings") + for field in ("sample_id", "group_id", "source_family", "temporal_family"): + relabeled[field] = f"spoofed-{field}" + with pytest.raises( + LeakageError, match="unavailable_provenance_record|protected_identity" + ): + assert_training_inputs_safe([], [relabeled], protected) + + broken_binding = copy.deepcopy(source) + broken_binding["samples"][0]["governance_binding"]["p3_item_ids"]["raw_image"] = ( + "0" * 20 + ) + with pytest.raises(LeakageError, match="provenance binding mismatch|P3 record"): + build_manifests(broken_binding, source_root=tmp_path) + + raw_path = Path(train[0]["content_path_bindings"]["raw_image"]["resolved_path"]) + raw_path.write_bytes(b"mutated after manifest creation") + with pytest.raises(LeakageError, match="record_path_hash_binding_mismatch"): + assert_training_inputs_safe([], train, protected) diff --git a/fixtures/accuracy/p4/protected-baseline-cases.json b/fixtures/accuracy/p4/protected-baseline-cases.json index cd214e49..340d71d5 100644 --- a/fixtures/accuracy/p4/protected-baseline-cases.json +++ b/fixtures/accuracy/p4/protected-baseline-cases.json @@ -1,5 +1,6 @@ { "schema_version": 2, + "portfolio_kind": "synthetic_contract", "portfolio_id": "geointel-p4-reference-harness-v2", "split_roles": [ "test", diff --git a/scripts/accuracy_phase4_evaluator.py b/scripts/accuracy_phase4_evaluator.py index b0740056..bc3b1b4a 100644 --- a/scripts/accuracy_phase4_evaluator.py +++ b/scripts/accuracy_phase4_evaluator.py @@ -1,7 +1,8 @@ """Fail-closed, task-aware metrics for the GeoIntel Phase 4 benchmark. -The bundled portfolio is a synthetic contract fixture. Nothing emitted by -this module is, by itself, evidence of production-model accuracy. +The evaluator accepts explicitly separated synthetic contract portfolios and +governed product-baseline portfolios. Nothing emitted by this module is, by +itself, evidence that inference provenance or production release gates passed. """ from __future__ import annotations @@ -14,7 +15,8 @@ from pathlib import Path from statistics import mean, median from typing import Any, Callable, Iterable -EVALUATOR_VERSION = "2.0.0" +PORTFOLIO_KINDS = {"synthetic_contract", "governed_product_baseline"} +EVALUATOR_VERSION = "2.1.0" REPORT_SCHEMA_VERSION = 2 SUBGROUP_MIN_CASE_SUPPORT = 5 CANONICAL_JSON_SPEC = ( @@ -46,6 +48,54 @@ ERROR_CODES = { "validation_false_positive": "D-VALIDATION-FP", "validation_false_negative": "D-VALIDATION-FN", "terrain_missing": "P-PARTIAL", + "raster_misclassification": "M-CLASS", + "raster_missing": "P-PARTIAL", + "boundary_error": "M-BOUNDARY", + "area_bias": "M-AREA-BIAS", + "miscalibrated": "M-MISCALIBRATED", +} +EXPECTED_PROTECTED_POLICY = { + "operating_point_selection_allowed": False, + "diagnostic_curves_select_operating_point": False, + "test_feedback_allowed": False, + "threshold_selection_source": "pre_registered_configuration_only", +} +REQUIRED_METADATA_STRING_FIELDS = ( + "region", + "municipality", + "urbanity", + "object_size", + "source", + "sensor", + "season", + "date", + "vegetation", + "occlusion", + "difficulty", + "context", +) +LINEAGE_SIDES = ("reference", "prediction") +LINEAGE_FIELDS = ("source_id", "source_version", "derivation") +NON_MEANINGFUL_TOKENS = {"", "unknown", "n/a", "na", "null", "tbd", "todo"} +BELGIUM_SCOPE_BOUNDS = (1.9, 49.4, 7.5, 52.1) +FAILURE_TAXONOMY = { + "M-FP-CONFUSER": "unmatched object or event prediction", + "M-FN-MISSED": "unmatched reference object or event", + "M-CLASS": "raster class differs from the valid reference class", + "M-BOUNDARY": "matched footprint has a measurable boundary deviation", + "M-AREA-BIAS": "matched footprint has a measurable relative area bias", + "M-MISCALIBRATED": "confidence diagnostic deviates from observed correctness", + "M-OOD": "error observed in an explicitly declared out-of-distribution case", + "D-VALIDATION-FP": "anomaly or severity reported without an exact reference match", + "D-VALIDATION-FN": "reference anomaly or severity not exactly reported", + "P-PARTIAL": "required prediction value is unavailable", +} +FAILURE_CONTEXTS = { + "tile_edge": "case metadata explicitly marks tile-edge context", + "high_confidence": ( + "false positive meets the highest frozen diagnostic confidence threshold" + ), + "out_of_distribution": "case metadata explicitly marks OOD context", } @@ -146,17 +196,62 @@ def _nonempty_mapping(value: Any, label: str) -> dict[str, Any]: return value +def _meaningful_string( + value: Any, + label: str, + *, + allow_not_applicable: bool = False, +) -> str: + if not isinstance(value, str) or value.strip().lower() in NON_MEANINGFUL_TOKENS: + raise ValueError(f"{label} must be a meaningful non-empty string") + normalized = value.strip() + if not allow_not_applicable and normalized.lower() == "not_applicable": + raise ValueError(f"{label} may not be not_applicable") + return normalized + + +def _validate_lineage(case: dict[str, Any]) -> None: + sample_id = case["sample_id"] + lineage = _nonempty_mapping(case.get("lineage"), f"{sample_id}: lineage") + missing_sides = sorted(set(LINEAGE_SIDES) - lineage.keys()) + if missing_sides: + raise ValueError(f"{sample_id}: lineage missing {missing_sides}") + for side in LINEAGE_SIDES: + entry = _nonempty_mapping(lineage.get(side), f"{sample_id}: lineage.{side}") + missing_fields = sorted(set(LINEAGE_FIELDS) - entry.keys()) + if missing_fields: + raise ValueError(f"{sample_id}: lineage.{side} missing {missing_fields}") + for field in LINEAGE_FIELDS: + _meaningful_string( + entry.get(field), + f"{sample_id}: lineage.{side}.{field}", + ) + + def _validate_case_common(case: dict[str, Any]) -> None: sample_id = case.get("sample_id") if not isinstance(sample_id, str) or not sample_id.strip(): raise ValueError("Every case requires a non-empty sample_id") if case.get("task") not in TASKS: raise ValueError(f"{sample_id}: unsupported task {case.get('task')!r}") - if not isinstance(case.get("metadata"), dict): - raise ValueError(f"{sample_id}: metadata must be an object") + metadata = _nonempty_mapping(case.get("metadata"), f"{sample_id}: metadata") + missing_metadata = sorted(set(REQUIRED_METADATA_STRING_FIELDS) - metadata.keys()) + if missing_metadata: + raise ValueError(f"{sample_id}: metadata missing {missing_metadata}") + for field in REQUIRED_METADATA_STRING_FIELDS: + _meaningful_string( + metadata.get(field), + f"{sample_id}: metadata.{field}", + allow_not_applicable=True, + ) + resolution = metadata.get("resolution_m") + if case["task"] == "geospatial_data_validation" and resolution is None: + pass + elif _finite_float(resolution, f"{sample_id}: metadata.resolution_m") <= 0: + raise ValueError(f"{sample_id}: metadata.resolution_m must be positive") if not isinstance(case.get("config"), dict): raise ValueError(f"{sample_id}: config must be an explicit object") - _nonempty_mapping(case.get("lineage"), f"{sample_id}: lineage") + _validate_lineage(case) def _stable_id(item: dict[str, Any], label: str) -> str: @@ -339,6 +434,8 @@ def detection_ap( reference = references[index] if not _class_compatible(prediction, reference): continue + if prediction.get("_sample_id") != reference.get("_sample_id"): + continue candidates.append( ( bbox_iou(prediction["bbox"], reference["bbox"]), @@ -510,6 +607,18 @@ def evaluate_object_detection(case: dict[str, Any]) -> dict[str, Any]: case["config"].get("match_iou"), f"{case['sample_id']}.config.match_iou", ) + diagnostic_thresholds = case["config"].get("fixed_diagnostic_risk_thresholds") + if diagnostic_thresholds is not None: + if not isinstance(diagnostic_thresholds, list) or not diagnostic_thresholds: + raise ValueError( + f"{case['sample_id']}: fixed diagnostic risk thresholds must be a " + "non-empty list" + ) + for index, value in enumerate(diagnostic_thresholds): + _probability( + value, + f"{case['sample_id']}.config.fixed_diagnostic_risk_thresholds[{index}]", + ) retained = [item for item in predictions if float(item["confidence"]) >= threshold] matches, false_positives, false_negatives = greedy_match( retained, @@ -538,6 +647,10 @@ def evaluate_object_detection(case: dict[str, Any]) -> dict[str, Any]: class_metrics["ap50"] = detection_ap( all_class_predictions, class_references, 0.5 ) + class_metrics["ap50_95"] = _mean_or_none( + detection_ap(all_class_predictions, class_references, 0.5 + step * 0.05) + for step in range(10) + ) per_class[str(class_value)] = class_metrics metrics = count_metrics(len(matches), len(false_positives), len(false_negatives)) metrics.update( @@ -555,9 +668,11 @@ def evaluate_object_detection(case: dict[str, Any]) -> dict[str, Any]: "coverage_risk": coverage_risk( predictions, references, match_iou, threshold ), + "map50": _mean_or_none(item["ap50"] for item in per_class.values()), + "map50_95": _mean_or_none(item["ap50_95"] for item in per_class.values()), } ) - return result( + evaluation = result( case, metrics, matches, @@ -571,12 +686,24 @@ def evaluate_object_detection(case: dict[str, Any]) -> dict[str, Any]: "threshold": threshold, }, ) - - -def polygon(item: dict[str, Any]): - from shapely.geometry import Polygon - - return Polygon(item["polygon"]) + calibration = metrics["calibration"] + if calibration["status"] == "computed" and calibration["ece"] > 1e-12: + evaluation["failures"].append( + failure_entry( + case, + "miscalibrated", + { + "ece": calibration["ece"], + "brier": calibration["brier"], + "prediction_support": len(retained), + "claim_boundary": ( + "diagnostic deviation only; not a representative " + "population-calibration claim" + ), + }, + ) + ) + return evaluation def distribution(values: list[float]) -> dict[str, Any]: @@ -594,6 +721,18 @@ def distribution(values: list[float]) -> dict[str, Any]: } +def _bounds_overlap( + left: tuple[float, float, float, float], + right: tuple[float, float, float, float], +) -> bool: + return not ( + left[2] < right[0] + or left[0] > right[2] + or left[3] < right[1] + or left[1] > right[3] + ) + + def _validate_metric_spatial_context(case: dict[str, Any]) -> dict[str, Any]: context = _nonempty_mapping( case.get("spatial_context"), @@ -629,32 +768,99 @@ def _validate_metric_spatial_context(case: dict[str, Any]) -> dict[str, Any]: for axis in crs.axis_info[:2] ): raise ValueError(f"{case['sample_id']}: CRS axes must use metres") + if crs.to_epsg() in {3395, 3857}: + raise ValueError( + f"{case['sample_id']}: Web/World Mercator is unsuitable for " + "benchmark area, boundary and distance metrics" + ) + area = crs.area_of_use + if area is None: + raise ValueError( + f"{case['sample_id']}: CRS area of use is unavailable; Belgian " + "metric suitability cannot be proven" + ) + crs_bounds = (area.west, area.south, area.east, area.north) + if not _bounds_overlap(crs_bounds, BELGIUM_SCOPE_BOUNDS): + raise ValueError( + f"{case['sample_id']}: CRS area of use does not overlap the " + "declared Belgium and Belgian North Sea product scope" + ) return context -def _validated_polygon(item: dict[str, Any], label: str): - coordinates = item.get("polygon") +def _validated_ring(coordinates: Any, label: str) -> list[list[float]]: if not isinstance(coordinates, list) or len(coordinates) < 4: raise ValueError( - f"{label}.polygon must contain a closed ring with at least four points" + f"{label} must contain a closed ring with at least four points" ) - normalized: list[tuple[float, float]] = [] + normalized: list[list[float]] = [] for index, point in enumerate(coordinates): if not isinstance(point, list) or len(point) != 2: - raise ValueError(f"{label}.polygon[{index}] must be an [x, y] pair") + raise ValueError(f"{label}[{index}] must be an [x, y] pair") normalized.append( - ( - _finite_float(point[0], f"{label}.polygon[{index}][0]"), - _finite_float(point[1], f"{label}.polygon[{index}][1]"), - ) + [ + _finite_float(point[0], f"{label}[{index}][0]"), + _finite_float(point[1], f"{label}[{index}][1]"), + ] ) if normalized[0] != normalized[-1]: - raise ValueError(f"{label}.polygon ring must be explicitly closed") - from shapely.geometry import Polygon + raise ValueError(f"{label} must be explicitly closed") + return normalized - geometry = Polygon(normalized) + +def _validated_polygon(item: dict[str, Any], label: str): + if "geometry" in item and "polygon" in item: + raise ValueError(f"{label} may not declare both geometry and polygon") + if "geometry" in item: + payload = item["geometry"] + if not isinstance(payload, dict): + raise ValueError(f"{label}.geometry must be a GeoJSON object") + geometry_type = payload.get("type") + coordinates = payload.get("coordinates") + if geometry_type == "Polygon": + if not isinstance(coordinates, list) or not coordinates: + raise ValueError(f"{label}.geometry Polygon requires rings") + normalized_coordinates: Any = [ + _validated_ring(ring, f"{label}.geometry.coordinates[{index}]") + for index, ring in enumerate(coordinates) + ] + elif geometry_type == "MultiPolygon": + if not isinstance(coordinates, list) or not coordinates: + raise ValueError( + f"{label}.geometry MultiPolygon requires polygon members" + ) + normalized_coordinates = [] + for polygon_index, polygon_coordinates in enumerate(coordinates): + if not isinstance(polygon_coordinates, list) or not polygon_coordinates: + raise ValueError( + f"{label}.geometry.coordinates[{polygon_index}] requires rings" + ) + normalized_coordinates.append( + [ + _validated_ring( + ring, + f"{label}.geometry.coordinates[{polygon_index}]" + f"[{ring_index}]", + ) + for ring_index, ring in enumerate(polygon_coordinates) + ] + ) + else: + raise ValueError(f"{label}.geometry.type must be Polygon or MultiPolygon") + normalized_payload = { + "type": geometry_type, + "coordinates": normalized_coordinates, + } + else: + normalized_payload = { + "type": "Polygon", + "coordinates": [_validated_ring(item.get("polygon"), f"{label}.polygon")], + } + from shapely.geometry import shape + + geometry = shape(normalized_payload) if geometry.is_empty or geometry.area <= 0 or not geometry.is_valid: - raise ValueError(f"{label}.polygon must be non-empty, positive-area and valid") + raise ValueError(f"{label}.geometry must be non-empty, positive-area and valid") return geometry @@ -679,16 +885,14 @@ def _validated_polygon_items( validated_references = [ dict( item, - geometry=_validated_polygon( - item, f"{case['sample_id']}.references[{index}]" - ), + _shape=_validated_polygon(item, f"{case['sample_id']}.references[{index}]"), ) for index, item in enumerate(references) ] validated_predictions = [ dict( item, - geometry=_validated_polygon( + _shape=_validated_polygon( item, f"{case['sample_id']}.predictions[{index}]" ), ) @@ -698,8 +902,8 @@ def _validated_polygon_items( def _polygon_overlap(prediction: dict[str, Any], reference: dict[str, Any]) -> float: - intersection = prediction["geometry"].intersection(reference["geometry"]).area - union = prediction["geometry"].union(reference["geometry"]).area + intersection = prediction["_shape"].intersection(reference["_shape"]).area + union = prediction["_shape"].union(reference["_shape"]).area return intersection / union if union > 0 else 0.0 @@ -753,13 +957,23 @@ def evaluate_footprint_segmentation(case: dict[str, Any]) -> dict[str, Any]: centroids = [] area_errors = [] for match in matches: - pred = by_prediction[match["prediction_id"]]["geometry"] - ref = by_reference[match["reference_id"]]["geometry"] + pred = by_prediction[match["prediction_id"]]["_shape"] + ref = by_reference[match["reference_id"]]["_shape"] intersection = pred.intersection(ref).area - dice.append(2 * intersection / (pred.area + ref.area)) - boundary.append(boundary_f1(pred, ref, tolerance)) - centroids.append(pred.centroid.distance(ref.centroid)) - area_errors.append((pred.area - ref.area) / ref.area) + dice_value = 2 * intersection / (pred.area + ref.area) + boundary_value = boundary_f1(pred, ref, tolerance) + centroid_value = pred.centroid.distance(ref.centroid) + area_error = (pred.area - ref.area) / ref.area + dice.append(dice_value) + boundary.append(boundary_value) + centroids.append(centroid_value) + area_errors.append(area_error) + match.update( + dice=dice_value, + boundary_f1=boundary_value, + centroid_distance_m=centroid_value, + relative_area_error=area_error, + ) metrics = count_metrics(len(matches), len(false_positives), len(false_negatives)) metrics.update( { @@ -773,13 +987,40 @@ def evaluate_footprint_segmentation(case: dict[str, Any]) -> dict[str, Any]: "spatial_context_validated": True, } ) - return result( + evaluation = result( case, metrics, matches, false_positives, false_negatives, ) + for match in matches: + if match["boundary_f1"] is not None and match["boundary_f1"] < 1 - 1e-12: + evaluation["failures"].append( + failure_entry( + case, + "boundary_error", + { + "prediction_id": match["prediction_id"], + "reference_id": match["reference_id"], + "boundary_f1": match["boundary_f1"], + "boundary_tolerance_m": tolerance, + }, + ) + ) + if abs(match["relative_area_error"]) > 1e-12: + evaluation["failures"].append( + failure_entry( + case, + "area_bias", + { + "prediction_id": match["prediction_id"], + "reference_id": match["reference_id"], + "relative_area_error": match["relative_area_error"], + }, + ) + ) + return evaluation def _validate_rectangular_grid( @@ -836,6 +1077,14 @@ def _validate_raster_side( ) for index, value in enumerate(transform) ) + a, b, _x_offset, d, e, _y_offset = normalized_transform + determinant = a * e - b * d + linear_scale = max(abs(a), abs(b), abs(d), abs(e), 1.0) + if abs(determinant) <= 1e-12 * linear_scale * linear_scale: + raise ValueError( + f"{case['sample_id']}: raster affine transform is singular or " + "numerically invalid" + ) shape = context["shape"] if ( not isinstance(shape, list) @@ -1195,9 +1444,22 @@ def evaluate_validation(case: dict[str, Any]) -> dict[str, Any]: observed_items = _normalized_anomalies(case, "observed_anomalies") expected = {item["code"]: item for item in expected_items} observed = {item["code"]: item for item in observed_items} - matched_codes = sorted(expected.keys() & observed.keys()) - false_positive_codes = sorted(observed.keys() - expected.keys()) - false_negative_codes = sorted(expected.keys() - observed.keys()) + common_codes = sorted(expected.keys() & observed.keys()) + matched_codes = [ + code + for code in common_codes + if expected[code]["severity"] == observed[code]["severity"] + ] + severity_mismatches = [ + { + "anomaly": code, + "expected_severity": expected[code]["severity"], + "observed_severity": observed[code]["severity"], + "reason": "severity_mismatch", + } + for code in common_codes + if expected[code]["severity"] != observed[code]["severity"] + ] matched = [ { "anomaly": code, @@ -1208,13 +1470,31 @@ def evaluate_validation(case: dict[str, Any]) -> dict[str, Any]: ] false_positives = [ {"anomaly": code, "severity": observed[code]["severity"]} - for code in false_positive_codes + for code in sorted(observed.keys() - expected.keys()) + ] + [ + { + "anomaly": item["anomaly"], + "severity": item["observed_severity"], + "expected_severity": item["expected_severity"], + "reason": "severity_mismatch", + } + for item in severity_mismatches ] false_negatives = [ {"anomaly": code, "severity": expected[code]["severity"]} - for code in false_negative_codes + for code in sorted(expected.keys() - observed.keys()) + ] + [ + { + "anomaly": item["anomaly"], + "severity": item["expected_severity"], + "observed_severity": item["observed_severity"], + "reason": "severity_mismatch", + } + for item in severity_mismatches ] metrics = count_metrics(len(matched), len(false_positives), len(false_negatives)) + metrics["severity_mismatch_count"] = len(severity_mismatches) + metrics["severity_mismatches"] = severity_mismatches metrics["blocker_or_critical_miss_count"] = sum( item["severity"] in {"blocker", "critical"} for item in false_negatives ) @@ -1259,6 +1539,7 @@ def result( "input_lineage": serializable(case["lineage"]), "reference_input_field": reference_field, "prediction_input_field": prediction_field, + "classes": serializable(case.get("classes")), "references": exact_references, "predictions_pre_filter": exact_predictions, "predictions_post_filter": post_filter, @@ -1297,7 +1578,7 @@ def result( def serializable(value: Any) -> Any: if isinstance(value, dict): return { - key: serializable(item) for key, item in value.items() if key != "geometry" + key: serializable(item) for key, item in value.items() if key != "_shape" } if isinstance(value, list): return [serializable(item) for item in value] @@ -1309,34 +1590,79 @@ def serializable(value: Any) -> Any: def failure_entry( case: dict[str, Any], kind: str, evidence: dict[str, Any] ) -> dict[str, Any]: - if case["task"] == "change_detection": - kind = ( - "event_false_positive" - if kind == "false_positive" - else "event_false_negative" - ) - elif case["task"] == "geospatial_data_validation": - kind = ( - "validation_false_positive" - if kind == "false_positive" - else "validation_false_negative" - ) - elif case["task"] == "terrain_interpretation": - kind = "terrain_missing" + base_kind = kind + if base_kind in {"false_positive", "false_negative"}: + if case["task"] == "change_detection": + kind = ( + "event_false_positive" + if base_kind == "false_positive" + else "event_false_negative" + ) + elif case["task"] == "geospatial_data_validation": + kind = ( + "validation_false_positive" + if base_kind == "false_positive" + else "validation_false_negative" + ) + elif case["task"] == "terrain_interpretation": + kind = "terrain_missing" + elif case["task"] == "raster_classification": + kind = ( + "raster_misclassification" + if evidence.get("reason") == "class_mismatch" + else "raster_missing" + ) + + contexts: list[str] = [] + secondary_error_codes: list[str] = [] + metadata = case.get("metadata", {}) + if metadata.get("tile_edge") is True: + contexts.append("tile_edge") + if metadata.get("ood") is True: + contexts.append("out_of_distribution") + secondary_error_codes.append("M-OOD") + confidence = evidence.get("confidence") + diagnostic_thresholds = case.get("config", {}).get( + "fixed_diagnostic_risk_thresholds" + ) + valid_thresholds = ( + [ + float(value) + for value in diagnostic_thresholds + if isinstance(value, (int, float)) + and not isinstance(value, bool) + and math.isfinite(float(value)) + and 0 <= float(value) <= 1 + ] + if isinstance(diagnostic_thresholds, list) + else [] + ) + if ( + base_kind == "false_positive" + and isinstance(confidence, (int, float)) + and not isinstance(confidence, bool) + and valid_thresholds + and float(confidence) >= max(valid_thresholds) + ): + contexts.append("high_confidence") + secondary_error_codes.append("M-MISCALIBRATED") + normalized_evidence = serializable(evidence) return { "failure_id": canonical_hash( { "sample_id": case["sample_id"], "kind": kind, - "evidence": serializable(evidence), + "evidence": normalized_evidence, } )[:20], "sample_id": case["sample_id"], "task": case["task"], "error_code": ERROR_CODES[kind], + "secondary_error_codes": sorted(set(secondary_error_codes)), + "contexts": sorted(set(contexts)), "kind": kind, - "metadata": case["metadata"], - "evidence": serializable(evidence), + "metadata": serializable(metadata), + "evidence": normalized_evidence, } @@ -1627,6 +1953,127 @@ def _aggregate_count_family( } +def _pooled_detection_inputs( + values: list[dict[str, Any]], +) -> tuple[ + list[dict[str, Any]], + list[dict[str, Any]], + list[dict[str, Any]], + list[dict[str, Any]], + list[str | int], +]: + references: list[dict[str, Any]] = [] + predictions_pre_filter: list[dict[str, Any]] = [] + predictions_post_filter: list[dict[str, Any]] = [] + matches: list[dict[str, Any]] = [] + classes_by_hash: dict[str, str | int] = {} + for value in sorted(values, key=lambda item: item.get("sample_id", "")): + sample_id = value.get("sample_id") + if not isinstance(sample_id, str) or not sample_id: + raise ValueError("Pooled detection metrics require a sample_id") + raw = _nonempty_mapping( + value.get("raw"), f"{sample_id}: pooled detection raw evidence" + ) + required_lists = ( + "references", + "predictions_pre_filter", + "predictions_post_filter", + "matches", + "classes", + ) + for field in required_lists: + if not isinstance(raw.get(field), list): + raise ValueError( + f"{sample_id}: pooled detection raw.{field} must be a list" + ) + for class_value in raw["classes"]: + classes_by_hash[canonical_hash(class_value)] = class_value + + def scoped(item: dict[str, Any], label: str) -> dict[str, Any]: + if not isinstance(item, dict): + raise ValueError(f"{sample_id}: pooled {label} must contain objects") + identifier = _stable_id(item, f"{sample_id}: pooled {label}") + return { + **item, + "id": f"{sample_id}::{identifier}", + "_sample_id": sample_id, + } + + references.extend(scoped(item, "references") for item in raw["references"]) + predictions_pre_filter.extend( + scoped(item, "predictions_pre_filter") + for item in raw["predictions_pre_filter"] + ) + predictions_post_filter.extend( + scoped(item, "predictions_post_filter") + for item in raw["predictions_post_filter"] + ) + for match in raw["matches"]: + if not isinstance(match, dict): + raise ValueError(f"{sample_id}: pooled matches must contain objects") + prediction_id = match.get("prediction_id") + if not isinstance(prediction_id, str) or not prediction_id: + raise ValueError( + f"{sample_id}: pooled match prediction_id must be non-empty" + ) + matches.append({**match, "prediction_id": f"{sample_id}::{prediction_id}"}) + return ( + references, + predictions_pre_filter, + predictions_post_filter, + matches, + [classes_by_hash[key] for key in sorted(classes_by_hash)], + ) + + +def _aggregate_detection(values: list[dict[str, Any]]) -> dict[str, Any]: + aggregation = _aggregate_count_family(values) + references, predictions, retained, matches, classes = _pooled_detection_inputs( + values + ) + per_class = {} + for class_value in classes: + class_references = [item for item in references if item["class"] == class_value] + class_predictions = [ + item for item in predictions if item["class"] == class_value + ] + ap50 = detection_ap(class_predictions, class_references, 0.5) + ap50_95 = _mean_or_none( + detection_ap(class_predictions, class_references, 0.5 + step * 0.05) + for step in range(10) + ) + per_class[str(class_value)] = { + "reference_count": len(class_references), + "prediction_count": len(class_predictions), + "ap50": ap50, + "ap50_95": ap50_95, + } + aggregation["micro"].update( + { + "ap50": detection_ap(predictions, references, 0.5), + "ap50_95": _mean_or_none( + detection_ap(predictions, references, 0.5 + step * 0.05) + for step in range(10) + ), + "map50": _mean_or_none(item["ap50"] for item in per_class.values()), + "map50_95": _mean_or_none(item["ap50_95"] for item in per_class.values()), + "per_class_ap": per_class, + "calibration": calibration_metrics(retained, matches), + "ranking_scope": ( + "predictions pooled across cases; class- and sample-aware matching; " + "no averaging of per-case AP" + ), + } + ) + aggregation["observation_support"].update( + { + "ranking_predictions": len(predictions), + "calibration_predictions": len(retained), + } + ) + return aggregation + + def _aggregate_raster(values: list[dict[str, Any]]) -> dict[str, Any]: metrics = [item["metrics"] for item in values] class_counts: dict[str, dict[str, int]] = defaultdict( @@ -1727,13 +2174,14 @@ def _aggregate_terrain(values: list[dict[str, Any]]) -> dict[str, Any]: def _aggregate_task(values: list[dict[str, Any]]) -> dict[str, Any]: task = values[0]["task"] - if task == "raster_classification": + if task == "object_detection": + aggregation = _aggregate_detection(values) + elif task == "raster_classification": aggregation = _aggregate_raster(values) elif task == "terrain_interpretation": aggregation = _aggregate_terrain(values) else: extras = { - "object_detection": ("ap50", "ap50_95"), "footprint_segmentation": ( "mean_iou", "mean_dice", @@ -1827,6 +2275,7 @@ def subgroup_report(results: list[dict[str, Any]]) -> dict[str, Any]: "vegetation", "occlusion", "difficulty", + "context", ) dimension_reports = {} any_insufficient = False @@ -1881,12 +2330,32 @@ def evaluate_cases(path: Path, allowed_sample_ids: set[str]) -> dict[str, Any]: portfolio = json.loads(path.read_text(encoding="utf-8")) if not isinstance(portfolio, dict): raise ValueError("The evaluation portfolio must be a JSON object") - claim_boundary = portfolio.get("claim_boundary") - if not isinstance(claim_boundary, str) or ( - "synthetic" not in claim_boundary.lower() - ): + schema_version = portfolio.get("schema_version") + if type(schema_version) is not int or schema_version != REPORT_SCHEMA_VERSION: raise ValueError( - "The portfolio claim boundary must explicitly state that it is synthetic" + f"schema_version must be exactly integer {REPORT_SCHEMA_VERSION}" + ) + _meaningful_string(portfolio.get("portfolio_id"), "portfolio_id") + protected_policy = portfolio.get("protected_policy") + if protected_policy != EXPECTED_PROTECTED_POLICY: + raise ValueError( + "protected_policy must exactly equal the frozen no-selection policy: " + f"{EXPECTED_PROTECTED_POLICY}" + ) + portfolio_kind = portfolio.get("portfolio_kind") + if portfolio_kind not in PORTFOLIO_KINDS: + raise ValueError(f"portfolio_kind must be one of {sorted(PORTFOLIO_KINDS)}") + claim_boundary = portfolio.get("claim_boundary") + claim_lower = claim_boundary.lower() if isinstance(claim_boundary, str) else "" + if portfolio_kind == "synthetic_contract": + if "synthetic" not in claim_lower: + raise ValueError( + "A synthetic_contract claim boundary must explicitly state that it is synthetic" + ) + elif "governed product baseline" not in claim_lower or "synthetic" in claim_lower: + raise ValueError( + "A governed_product_baseline claim boundary must explicitly state " + "'governed product baseline' and must not describe the portfolio as synthetic" ) split_roles = portfolio.get("split_roles") if split_roles not in (["test"], ["test", "background-test"]): @@ -1895,17 +2364,18 @@ def evaluate_cases(path: Path, allowed_sample_ids: set[str]) -> dict[str, Any]: ) if "challenge_cases" in portfolio or "challenge_labels" in portfolio: raise ValueError("Challenge cases and labels must remain sealed and absent") - selection_policy = portfolio.get("selection_policy") - if not isinstance(selection_policy, str) or not selection_policy.strip(): - raise ValueError("The portfolio requires an explicit selection policy") + selection_policy = _meaningful_string( + portfolio.get("selection_policy"), "selection_policy" + ) declared_portfolio_lineage = _nonempty_mapping( portfolio.get("portfolio_lineage"), "portfolio_lineage", ) for field in ("origin", "source_path", "version"): - value = declared_portfolio_lineage.get(field) - if not isinstance(value, str) or not value.strip(): - raise ValueError(f"portfolio_lineage.{field} must be a non-empty string") + _meaningful_string( + declared_portfolio_lineage.get(field), + f"portfolio_lineage.{field}", + ) cases = portfolio.get("cases") if not isinstance(cases, list) or not all(isinstance(item, dict) for item in cases): raise ValueError("The portfolio cases must be an object list") @@ -1923,6 +2393,13 @@ def evaluate_cases(path: Path, allowed_sample_ids: set[str]) -> dict[str, Any]: if duplicates: raise ValueError(f"Duplicate protected case ids: {duplicates}") case_ids = set(identifiers) + observed_tasks = {item["task"] for item in cases} + if portfolio_kind == "governed_product_baseline" and observed_tasks != TASKS: + raise ValueError( + "A governed_product_baseline must cover every evaluator task family: " + f"missing={sorted(TASKS - observed_tasks)}, " + f"unexpected={sorted(observed_tasks - TASKS)}" + ) unexpected = sorted(case_ids - allowed_sample_ids) missing = sorted(allowed_sample_ids - case_ids) if unexpected or missing: @@ -1939,6 +2416,7 @@ def evaluate_cases(path: Path, allowed_sample_ids: set[str]) -> dict[str, Any]: "declared": serializable(declared_portfolio_lineage), "portfolio_id": portfolio.get("portfolio_id"), "portfolio_schema_version": portfolio.get("schema_version"), + "portfolio_kind": portfolio_kind, "portfolio_file_sha256": file_hash, "portfolio_canonical_json_sha256": portfolio_canonical_hash, "source_path": declared_portfolio_lineage["source_path"], @@ -1947,6 +2425,13 @@ def evaluate_cases(path: Path, allowed_sample_ids: set[str]) -> dict[str, Any]: } for item in results: item["raw"]["portfolio_lineage"] = portfolio_lineage + results_by_task: dict[str, list[dict[str, Any]]] = defaultdict(list) + for item in results: + results_by_task[item["task"]].append(item) + portfolio_metrics = { + task: _aggregate_task(task_results) + for task, task_results in sorted(results_by_task.items()) + } failures = sorted( (failure for item in results for failure in item["failures"]), key=lambda item: item["failure_id"], @@ -1956,6 +2441,7 @@ def evaluate_cases(path: Path, allowed_sample_ids: set[str]) -> dict[str, Any]: "schema_version": REPORT_SCHEMA_VERSION, "evaluator_version": EVALUATOR_VERSION, "portfolio_id": portfolio["portfolio_id"], + "portfolio_kind": portfolio_kind, "portfolio_file_sha256": file_hash, "portfolio_canonical_json_sha256": portfolio_canonical_hash, "claim_boundary": claim_boundary, @@ -1978,5 +2464,11 @@ def evaluate_cases(path: Path, allowed_sample_ids: set[str]) -> dict[str, Any]: "results": results, "subgroups": subgroup_report(results), "failures": failures, + "failure_taxonomy": { + "primary_codes": FAILURE_TAXONOMY, + "contexts": FAILURE_CONTEXTS, + "claim_boundary": "diagnostic classification; not a release decision", + }, + "portfolio_metrics": portfolio_metrics, "results_canonical_json_sha256": results_hash, } diff --git a/scripts/generate_accuracy_phase4_splits.py b/scripts/generate_accuracy_phase4_splits.py index 2348d388..4576ec58 100644 --- a/scripts/generate_accuracy_phase4_splits.py +++ b/scripts/generate_accuracy_phase4_splits.py @@ -8,23 +8,58 @@ import hashlib import json import math import re +import unicodedata from collections import Counter, defaultdict -from datetime import date +from datetime import date, datetime from pathlib import Path from typing import Any, Iterable from pyproj import CRS SCHEMA_VERSION = 1 -GENERATOR_VERSION = "1.1.0" +GENERATOR_VERSION = "1.3.0" TRAIN_SPLITS = {"train"} SELECTION_SPLITS = {"val", "calibration"} DEVELOPMENT_SPLITS = TRAIN_SPLITS | SELECTION_SPLITS PROTECTED_SPLITS = {"test", "background-test", "challenge"} ALL_SPLITS = DEVELOPMENT_SPLITS | PROTECTED_SPLITS -NORMATIVE_SPLITS = {"train", "val", "calibration", "test", "background-test"} +MANDATORY_SPLIT_ROLES = ( + "train", + "val", + "calibration", + "test", + "background-test", + "challenge", +) +NORMATIVE_SPLITS = frozenset(MANDATORY_SPLIT_ROLES) +EVALUATOR_PROTECTED_SPLITS = frozenset({"test", "background-test"}) +MIN_INDEPENDENCE_BUFFER_M = 2000.0 +MIN_PERCEPTUAL_HAMMING_THRESHOLD = 4 +MIN_LABEL_GEOMETRY_HAMMING_THRESHOLD = 2 +TRUSTED_FIXTURE_POLICY = { + "dataset_version": "geointel-p4-harness-fixture-v2", + "claim_boundary": ( + "Synthetic contract fixtures for evaluator regression only; " + "never production accuracy evidence." + ), + "policy_id": "geointel-p4-synthetic-fixture-v2", +} SHA256_PATTERN = re.compile(r"^[0-9a-f]{64}$") FINGERPRINT_PATTERN = re.compile(r"^[0-9a-f]{16}$") +P3_ITEM_ID_PATTERN = re.compile(r"^[0-9a-f]{20}$") +PROVENANCE_MANIFEST_TYPE = "geointel_phase4_source_provenance" +ASSET_BINDINGS = { + "raw_image": ("raw_image_path", "raw_image_sha256"), + "processed_image": ("processed_image_path", "processed_image_sha256"), + "label": ("label_path", "label_sha256"), + "label_geometry": ("label_geometry_path", "label_geometry_hash"), +} +FIXTURE_ACTIVATION_FIELDS = { + "trusted_fixture_mode", + "fixture_mode", + "allow_synthetic_fixture", + "test_fixture_mode", +} class LeakageError(ValueError): @@ -41,6 +76,71 @@ def canonical_hash(value: Any) -> str: return hashlib.sha256(canonical_bytes(value)).hexdigest() +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def canonical_identifier(value: Any, *, field: str, sample_id: str) -> str: + if not isinstance(value, str): + raise LeakageError(f"{sample_id}: {field} must be a string identifier") + if any(unicodedata.category(character) == "Cc" for character in value): + raise LeakageError(f"{sample_id}: {field} contains control characters") + normalized = " ".join(unicodedata.normalize("NFKC", value).split()).casefold() + if not normalized: + raise LeakageError(f"{sample_id}: {field} must be non-empty") + return normalized + + +def normalized_identifier_list(values: Any, *, field: str, sample_id: str) -> list[str]: + if not isinstance(values, list): + raise LeakageError(f"{sample_id}: {field} must be a list") + normalized: list[str] = [] + originals_by_identity: dict[str, set[str]] = defaultdict(set) + for value in values: + identity = canonical_identifier(value, field=field, sample_id=sample_id) + originals_by_identity[identity].add(str(value)) + normalized.append(identity) + ambiguous = { + identity: sorted(originals) + for identity, originals in originals_by_identity.items() + if len(originals) > 1 or normalized.count(identity) > 1 + } + if ambiguous: + raise LeakageError( + f"{sample_id}: {field} contains ambiguous canonical identities: {ambiguous}" + ) + return sorted(normalized) + + +def validated_policy_thresholds(source: dict[str, Any]) -> tuple[float, int, int]: + try: + buffer_m = float(source.get("independence_buffer_m")) + perceptual = int(source.get("perceptual_hamming_threshold")) + geometry = int(source.get("label_geometry_hamming_threshold")) + except (TypeError, ValueError) as exc: + raise LeakageError("Split isolation thresholds must be numeric") from exc + if not math.isfinite(buffer_m) or buffer_m < MIN_INDEPENDENCE_BUFFER_M: + raise LeakageError( + "independence_buffer_m cannot weaken the code-owned minimum " + f"of {MIN_INDEPENDENCE_BUFFER_M:g} m" + ) + if not MIN_PERCEPTUAL_HAMMING_THRESHOLD <= perceptual < 64: + raise LeakageError( + "perceptual_hamming_threshold cannot weaken the code-owned minimum " + f"of {MIN_PERCEPTUAL_HAMMING_THRESHOLD}" + ) + if not MIN_LABEL_GEOMETRY_HAMMING_THRESHOLD <= geometry < 64: + raise LeakageError( + "label_geometry_hamming_threshold cannot weaken the code-owned minimum " + f"of {MIN_LABEL_GEOMETRY_HAMMING_THRESHOLD}" + ) + return buffer_m, perceptual, geometry + + def write_json(path: Path, payload: Any) -> None: path.parent.mkdir(parents=True, exist_ok=True) content = json.dumps(payload, ensure_ascii=False, indent=2, sort_keys=True) + "\n" @@ -84,7 +184,441 @@ def bbox_distance(left: list[float], right: list[float]) -> float: return math.hypot(dx, dy) -def normalized_sample(sample: dict[str, Any]) -> dict[str, Any]: +def is_trusted_fixture_source(source: dict[str, Any]) -> bool: + return ( + source.get("dataset_version") == TRUSTED_FIXTURE_POLICY["dataset_version"] + and source.get("claim_boundary") == TRUSTED_FIXTURE_POLICY["claim_boundary"] + ) + + +def _resolve_evidence_path(value: Any, source_root: Path | None) -> Path: + if not isinstance(value, str) or not value.strip(): + raise LeakageError("Governance evidence path must be non-empty") + path = Path(value) + if not path.is_absolute(): + if source_root is None: + raise LeakageError( + "Relative governance evidence paths require the source manifest root" + ) + path = source_root / path + return path.resolve() + + +def _required_string(value: Any, *, field: str) -> str: + if not isinstance(value, str) or not value.strip() or value != value.strip(): + raise LeakageError(f"{field} must be a non-empty, trimmed string") + if any(unicodedata.category(character) == "Cc" for character in value): + raise LeakageError(f"{field} contains control characters") + return value + + +def _canonical_bound_path(value: Any, *, field: str) -> str: + return Path(_required_string(value, field=field)).as_posix() + + +def _load_evidence_object( + binding: Any, + *, + key: str, + source_root: Path | None, + from_protected_manifest: bool = False, +) -> tuple[dict[str, Any], dict[str, Any]]: + required = ( + {"path", "sha256", "resolved_path"} + if from_protected_manifest + else { + "path", + "sha256", + } + ) + if not isinstance(binding, dict) or set(binding) != required: + raise LeakageError( + f"governance_evidence.{key} must contain exactly {sorted(required)}" + ) + expected = require_hex( + "", + f"governance_evidence.{key}.sha256", + binding.get("sha256"), + SHA256_PATTERN, + ) + path_value = ( + binding["resolved_path"] if from_protected_manifest else binding["path"] + ) + path = _resolve_evidence_path(path_value, source_root) + if not path.is_file(): + raise LeakageError(f"Governance evidence is unavailable: {path}") + actual = sha256_file(path) + if actual != expected: + raise LeakageError( + f"Governance evidence checksum mismatch for {key}: {actual} != {expected}" + ) + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + raise LeakageError(f"Governance evidence is not readable JSON: {path}") from exc + if not isinstance(payload, dict) or not payload: + raise LeakageError( + f"Governance evidence must be a non-empty JSON object: {path}" + ) + return payload, { + "path": str(binding["path"]), + "resolved_path": str(path), + "sha256": actual, + } + + +def _validate_p3_manifest(payload: dict[str, Any]) -> dict[str, dict[str, Any]]: + required_root = { + "schema_version", + "scan_id", + "scanner_version", + "completed_at", + "items", + "reconciliation", + } + missing = sorted(required_root - set(payload)) + if missing or payload.get("schema_version") != 1: + raise LeakageError(f"P3 scan manifest has an invalid schema; missing={missing}") + _required_string(payload["scan_id"], field="P3 scan_id") + _required_string(payload["scanner_version"], field="P3 scanner_version") + try: + completed = datetime.fromisoformat( + _required_string(payload["completed_at"], field="P3 completed_at") + ) + except ValueError as exc: + raise LeakageError("P3 completed_at must be ISO-8601") from exc + if completed.tzinfo is None: + raise LeakageError("P3 completed_at must include a timezone") + reconciliation = payload["reconciliation"] + count_fields = ("examined", "skipped", "unreachable", "inventory_total") + if ( + not isinstance(reconciliation, dict) + or reconciliation.get("reconciles") is not True + ): + raise LeakageError("P3 reconciliation must explicitly pass") + counts: dict[str, int] = {} + for field in count_fields: + value = reconciliation.get(field) + if isinstance(value, bool) or not isinstance(value, int) or value < 0: + raise LeakageError(f"P3 reconciliation.{field} is invalid") + counts[field] = value + if sum(counts[field] for field in count_fields[:3]) != counts["inventory_total"]: + raise LeakageError("P3 reconciliation counts do not add up") + items = payload["items"] + if ( + not isinstance(items, list) + or not items + or len(items) != counts["inventory_total"] + ): + raise LeakageError("P3 item count does not match the reconciled inventory") + required_item = { + "item_id", + "path", + "sha256", + "size_bytes", + "status", + "read_status", + "recommended_action", + "empty_content", + "schema_conformity", + "anomalies", + } + indexed: dict[str, dict[str, Any]] = {} + for position, item in enumerate(items): + if not isinstance(item, dict) or not required_item <= set(item): + raise LeakageError(f"P3 item {position} has an invalid schema") + item_id = require_hex( + f"", "item_id", item["item_id"], P3_ITEM_ID_PATTERN + ) + if item_id in indexed: + raise LeakageError(f"P3 scan manifest contains duplicate item_id {item_id}") + _canonical_bound_path(item["path"], field=f"P3 item {item_id}.path") + size = item["size_bytes"] + if size is not None and ( + isinstance(size, bool) or not isinstance(size, int) or size < 0 + ): + raise LeakageError(f"P3 item {item_id}.size_bytes is invalid") + if item["sha256"] is not None: + require_hex(item_id, "sha256", item["sha256"], SHA256_PATTERN) + if not isinstance(item["anomalies"], list): + raise LeakageError(f"P3 item {item_id}.anomalies must be a list") + indexed[item_id] = item + return indexed + + +def _validate_provenance_manifest( + payload: dict[str, Any], +) -> dict[str, dict[str, Any]]: + root_fields = { + "schema_version", + "manifest_type", + "status", + "records", + "records_canonical_json_sha256", + } + if ( + set(payload) != root_fields + or payload.get("schema_version") != 1 + or payload.get("manifest_type") != PROVENANCE_MANIFEST_TYPE + or payload.get("status") != "pass" + ): + raise LeakageError("Source provenance manifest has an invalid schema or status") + records = payload["records"] + expected_hash = require_hex( + "", + "records_canonical_json_sha256", + payload["records_canonical_json_sha256"], + SHA256_PATTERN, + ) + if ( + not isinstance(records, list) + or not records + or canonical_hash(records) != expected_hash + ): + raise LeakageError("Source provenance records are empty or checksum-invalid") + record_fields = { + "record_id", + "sample_id", + "status", + "lineage_status", + "training_allowed", + "perceptual_image_hash", + "label_geometry_fingerprint", + "assets", + } + asset_fields = {"path", "sha256", "size_bytes", "p3_item_id"} + indexed: dict[str, dict[str, Any]] = {} + seen_samples: set[str] = set() + for position, record in enumerate(records): + if not isinstance(record, dict) or set(record) != record_fields: + raise LeakageError( + f"Source provenance record {position} has an invalid schema" + ) + record_id = _required_string( + record["record_id"], field=f"provenance record {position}.record_id" + ) + sample_id = canonical_identifier( + record["sample_id"], field="sample_id", sample_id=record_id + ) + if ( + sample_id != record["sample_id"] + or record_id in indexed + or sample_id in seen_samples + ): + raise LeakageError("Source provenance manifest has ambiguous identities") + if ( + record["status"] != "accepted" + or record["lineage_status"] != "complete" + or record["training_allowed"] is not True + ): + raise LeakageError(f"Source provenance record {record_id} is not accepted") + for field in ("perceptual_image_hash", "label_geometry_fingerprint"): + require_hex(sample_id, field, record[field], FINGERPRINT_PATTERN) + assets = record["assets"] + if not isinstance(assets, dict) or set(assets) != set(ASSET_BINDINGS): + raise LeakageError( + f"Source provenance record {record_id} has incomplete assets" + ) + for role, asset in assets.items(): + if not isinstance(asset, dict) or set(asset) != asset_fields: + raise LeakageError( + f"Source provenance record {record_id}.{role} has an invalid schema" + ) + _canonical_bound_path(asset["path"], field=f"{record_id}.{role}.path") + require_hex(sample_id, f"{role}.sha256", asset["sha256"], SHA256_PATTERN) + size = asset["size_bytes"] + if isinstance(size, bool) or not isinstance(size, int) or size < 0: + raise LeakageError(f"{record_id}.{role}.size_bytes is invalid") + require_hex( + sample_id, + f"{role}.p3_item_id", + asset["p3_item_id"], + P3_ITEM_ID_PATTERN, + ) + indexed[record_id] = record + seen_samples.add(sample_id) + return indexed + + +def _validated_governance_bundle( + evidence: Any, + *, + source_root: Path | None, + from_protected_manifest: bool = False, +) -> tuple[dict[str, Any], dict[str, dict[str, Any]], dict[str, dict[str, Any]]]: + keys = {"p3_scan_manifest", "source_provenance_manifest"} + if not isinstance(evidence, dict) or set(evidence) != keys: + raise LeakageError(f"governance_evidence must contain exactly {sorted(keys)}") + p3, p3_binding = _load_evidence_object( + evidence["p3_scan_manifest"], + key="p3_scan_manifest", + source_root=source_root, + from_protected_manifest=from_protected_manifest, + ) + provenance, provenance_binding = _load_evidence_object( + evidence["source_provenance_manifest"], + key="source_provenance_manifest", + source_root=source_root, + from_protected_manifest=from_protected_manifest, + ) + p3_items = _validate_p3_manifest(p3) + provenance_records = _validate_provenance_manifest(provenance) + verified = { + "p3_scan_manifest": { + **p3_binding, + "scan_id": p3["scan_id"], + "scanner_version": p3["scanner_version"], + }, + "source_provenance_manifest": { + **provenance_binding, + "manifest_type": provenance["manifest_type"], + "records_canonical_json_sha256": provenance[ + "records_canonical_json_sha256" + ], + }, + } + return verified, p3_items, provenance_records + + +def validate_source_trust( + source: dict[str, Any], + source_root: Path | None, + *, + trusted_fixture_mode: bool = False, +) -> dict[str, Any]: + activation = sorted(FIXTURE_ACTIVATION_FIELDS & set(source)) + if activation: + raise LeakageError( + f"Fixture mode cannot be enabled by source metadata: {activation}" + ) + if trusted_fixture_mode: + if not is_trusted_fixture_source(source): + raise LeakageError( + "Explicit fixture mode only accepts the code-owned fixture policy" + ) + return { + "mode": "synthetic_fixture", + "policy_id": TRUSTED_FIXTURE_POLICY["policy_id"], + "production_accuracy_use_allowed": False, + } + if is_trusted_fixture_source(source): + raise LeakageError( + "Synthetic fixture source requires explicit trusted_fixture_mode=True" + ) + verified, p3_items, provenance_records = _validated_governance_bundle( + source.get("governance_evidence"), source_root=source_root + ) + return { + "mode": "governed_production", + "verified_evidence": verified, + "production_accuracy_use_allowed": True, + "_p3_items_by_id": p3_items, + "_provenance_records_by_id": provenance_records, + } + + +def _verified_content_hashes( + sample: dict[str, Any], + *, + sample_id: str, + source_root: Path | None, + source_trust: dict[str, Any], +) -> tuple[dict[str, str], dict[str, dict[str, Any]]]: + statuses: dict[str, str] = {} + bindings: dict[str, dict[str, Any]] = {} + provenance_record: dict[str, Any] | None = None + provenance_id: str | None = None + p3_ids: dict[str, Any] = {} + if source_trust["mode"] == "governed_production": + governance = sample.get("governance_binding") + required = {"source_provenance_record_id", "p3_item_ids"} + if not isinstance(governance, dict) or set(governance) != required: + raise LeakageError( + f"{sample_id}: governance_binding must contain exactly {sorted(required)}" + ) + provenance_id = _required_string( + governance["source_provenance_record_id"], + field=f"{sample_id}.source_provenance_record_id", + ) + p3_ids = governance["p3_item_ids"] + if not isinstance(p3_ids, dict) or set(p3_ids) != set(ASSET_BINDINGS): + raise LeakageError(f"{sample_id}: p3_item_ids must bind every asset") + provenance_record = source_trust["_provenance_records_by_id"].get(provenance_id) + if provenance_record is None or provenance_record["sample_id"] != sample_id: + raise LeakageError(f"{sample_id}: exact provenance record is unavailable") + for field in ("perceptual_image_hash", "label_geometry_fingerprint"): + if provenance_record[field] != str(sample[field]).lower(): + raise LeakageError(f"{sample_id}: provenance {field} mismatch") + + for role, (path_field, hash_field) in ASSET_BINDINGS.items(): + expected = require_hex( + sample_id, hash_field, sample[hash_field], SHA256_PATTERN + ) + path_value = sample.get(path_field) + if path_value is None: + if source_trust["mode"] == "governed_production": + raise LeakageError( + f"{sample_id}: governed production requires accessible {path_field}" + ) + statuses[hash_field] = "synthetic_fixture_bound" + continue + path = _resolve_evidence_path(path_value, source_root) + if not path.is_file(): + raise LeakageError(f"{sample_id}: content path is unavailable: {path}") + actual = sha256_file(path) + size = path.stat().st_size + if actual != expected: + raise LeakageError( + f"{sample_id}: {hash_field} checksum mismatch: {actual} != {expected}" + ) + statuses[hash_field] = "recomputed_from_accessible_file" + if source_trust["mode"] != "governed_production": + continue + assert provenance_record is not None and provenance_id is not None + p3_id = require_hex( + sample_id, f"p3_item_ids.{role}", p3_ids[role], P3_ITEM_ID_PATTERN + ) + asset = provenance_record["assets"][role] + canonical_path = _canonical_bound_path(path_value, field=path_field) + if ( + _canonical_bound_path(asset["path"], field=f"{provenance_id}.{role}.path") + != canonical_path + or asset["sha256"] != actual + or asset["size_bytes"] != size + or asset["p3_item_id"] != p3_id + ): + raise LeakageError(f"{sample_id}: provenance binding mismatch for {role}") + p3_item = source_trust["_p3_items_by_id"].get(p3_id) + if p3_item is None or ( + _canonical_bound_path(p3_item["path"], field=f"P3 {p3_id}.path") + != canonical_path + or p3_item["sha256"] != actual + or p3_item["size_bytes"] != size + or p3_item["status"] != "examined" + or p3_item["read_status"] != "readable" + or p3_item["recommended_action"] != "accept" + or p3_item["empty_content"] is not False + or p3_item["schema_conformity"] not in {"conformant", "not_applicable"} + or p3_item["anomalies"] + ): + raise LeakageError(f"{sample_id}: P3 record is not accepted for {role}") + bindings[role] = { + "path": str(path_value), + "resolved_path": str(path), + "sha256": actual, + "size_bytes": size, + "p3_item_id": p3_id, + "source_provenance_record_id": provenance_id, + } + return statuses, bindings + + +def normalized_sample( + sample: dict[str, Any], + *, + source_root: Path | None = None, + source_trust: dict[str, Any] | None = None, +) -> dict[str, Any]: required = { "sample_id", "task", @@ -110,42 +644,60 @@ def normalized_sample(sample: dict[str, Any]) -> dict[str, Any]: raise LeakageError( f"{sample.get('sample_id', '')}: missing fields {missing}" ) - split = str(sample["split"]) + raw_sample_id = str(sample["sample_id"]) + sample_id = canonical_identifier( + sample["sample_id"], field="sample_id", sample_id=raw_sample_id + ) + split = canonical_identifier(sample["split"], field="split", sample_id=sample_id) if split not in ALL_SPLITS: - raise LeakageError(f"{sample['sample_id']}: unsupported split {split!r}") + raise LeakageError(f"{sample_id}: unsupported split {split!r}") bbox = sample["bbox"] if not isinstance(bbox, list) or len(bbox) != 4: - raise LeakageError(f"{sample['sample_id']}: bbox must contain four values") - values = [float(value) for value in bbox] + raise LeakageError(f"{sample_id}: bbox must contain four values") + try: + values = [float(value) for value in bbox] + except (TypeError, ValueError) as exc: + raise LeakageError(f"{sample_id}: bbox values must be numeric") from exc if ( not all(math.isfinite(value) for value in values) or values[0] >= values[2] or values[1] >= values[3] ): - raise LeakageError(f"{sample['sample_id']}: invalid bbox") + raise LeakageError(f"{sample_id}: invalid bbox") + if source_trust is None: + raise LeakageError(f"{sample_id}: explicit source trust is required") + trust = source_trust result = dict(sample) + result["sample_id"] = sample_id + result["task"] = canonical_identifier( + sample["task"], field="task", sample_id=sample_id + ) + result["split"] = split result["bbox"] = values + hash_verification, content_bindings = _verified_content_hashes( + sample, + sample_id=sample_id, + source_root=source_root, + source_trust=trust, + ) + result["content_hash_verification"] = hash_verification + if content_bindings: + result["content_path_bindings"] = content_bindings for field in ( "raw_image_sha256", "processed_image_sha256", "label_sha256", "label_geometry_hash", ): - result[field] = require_hex( - str(sample["sample_id"]), field, sample[field], SHA256_PATTERN - ) + result[field] = require_hex(sample_id, field, sample[field], SHA256_PATTERN) for field in ("perceptual_image_hash", "label_geometry_fingerprint"): result[field] = require_hex( - str(sample["sample_id"]), field, sample[field], FINGERPRINT_PATTERN + sample_id, field, sample[field], FINGERPRINT_PATTERN ) for field in ("object_ids", "native_feature_ids"): - if not isinstance(sample[field], list) or any( - not str(value) for value in sample[field] - ): - raise LeakageError( - f"{sample['sample_id']}: {field} must be a list of non-empty identities" - ) - result[field] = sorted({str(value) for value in sample[field]}) + result[field] = normalized_identifier_list( + sample[field], field=field, sample_id=sample_id + ) for field in ( "group_id", "source_family", @@ -153,17 +705,23 @@ def normalized_sample(sample: dict[str, Any]) -> dict[str, Any]: "parent_raster_id", "acquisition_id", ): - if not str(sample[field]).strip(): - raise LeakageError(f"{sample['sample_id']}: {field} must be non-empty") - result[field] = str(sample[field]) + result[field] = canonical_identifier( + sample[field], field=field, sample_id=sample_id + ) try: result["acquisition_date"] = date.fromisoformat( str(sample["acquisition_date"]) ).isoformat() - except ValueError as exc: - raise LeakageError( - f"{sample['sample_id']}: acquisition_date must be ISO-8601" - ) from exc + except (TypeError, ValueError) as exc: + raise LeakageError(f"{sample_id}: acquisition_date must be ISO-8601") from exc + if trust["mode"] == "governed_production": + binding = sample["governance_binding"] + result["governance_binding"] = { + "source_provenance_record_id": binding["source_provenance_record_id"], + "p3_item_ids": { + role: binding["p3_item_ids"][role] for role in ASSET_BINDINGS + }, + } record_without_split = { key: value for key, value in result.items() if key != "split" } @@ -171,10 +729,19 @@ def normalized_sample(sample: dict[str, Any]) -> dict[str, Any]: return result -def assign_split_roles(source: dict[str, Any]) -> list[dict[str, Any]]: +def assign_split_roles( + source: dict[str, Any], + *, + source_root: Path | None = None, + source_trust: dict[str, Any] | None = None, +) -> list[dict[str, Any]]: """Assign complete leakage groups deterministically, or validate a frozen assignment.""" raw_samples = [dict(item) for item in source.get("samples", [])] - mode = str(source.get("assignment_mode") or "preassigned") + mode = canonical_identifier( + source.get("assignment_mode") or "preassigned", + field="assignment_mode", + sample_id="", + ) split_presence = [bool(item.get("split")) for item in raw_samples] if mode == "preassigned": if raw_samples and not all(split_presence): @@ -187,19 +754,24 @@ def assign_split_roles(source: dict[str, Any]) -> list[dict[str, Any]]: "deterministic_grouped mode refuses partially or fully preassigned splits" ) config = source.get("split_assignment") or {} - roles = list( - config.get("roles") - or ["train", "val", "calibration", "test", "background-test"] - ) - if not roles or len(set(roles)) != len(roles) or set(roles) - ALL_SPLITS: - raise LeakageError("split_assignment.roles must be unique supported roles") + requested_roles = config.get("roles") + if requested_roles is not None and tuple(requested_roles) != MANDATORY_SPLIT_ROLES: + raise LeakageError( + "split_assignment.roles cannot change the code-owned mandatory role order" + ) + roles = list(MANDATORY_SPLIT_ROLES) weights = { role: float((config.get("weights") or {}).get(role, 1.0)) for role in roles } if any(not math.isfinite(value) or value <= 0 for value in weights.values()): raise LeakageError("split_assignment weights must be positive finite values") placeholder = [ - normalized_sample({**item, "split": "train"}) for item in raw_samples + normalized_sample( + {**item, "split": "train"}, + source_root=source_root, + source_trust=source_trust, + ) + for item in raw_samples ] parent = list(range(len(placeholder))) @@ -219,6 +791,7 @@ def assign_split_roles(source: dict[str, Any]) -> list[dict[str, Any]]: "group_id", "source_family", "temporal_family", + "acquisition_date", "raw_image_sha256", "processed_image_sha256", "label_sha256", @@ -238,9 +811,7 @@ def assign_split_roles(source: dict[str, Any]) -> list[dict[str, Any]]: union(index, seen[identity]) else: seen[identity] = index - image_threshold = int(source.get("perceptual_hamming_threshold", 4)) - geometry_threshold = int(source.get("label_geometry_hamming_threshold", 2)) - buffer_m = float(source.get("independence_buffer_m") or 0) + buffer_m, image_threshold, geometry_threshold = validated_policy_thresholds(source) for index, left in enumerate(placeholder): for right_index, right in enumerate(placeholder[index + 1 :], start=index + 1): if ( @@ -304,6 +875,51 @@ def assign_split_roles(source: dict[str, Any]) -> list[dict[str, Any]]: assigned_counts[role] += len(indices) for stratum in strata: stratum_counts[stratum][role] += len(indices) + group_role = {indices[0]: assignment[indices[0]] for indices in groups} + role_group_counts = Counter(group_role.values()) + for task in sorted({sample["task"] for sample in placeholder}): + task_groups = [ + indices + for indices in groups + if any(placeholder[index]["task"] == task for index in indices) + ] + if any( + assignment[indices[0]] in EVALUATOR_PROTECTED_SPLITS + for indices in task_groups + ): + continue + movable = [ + indices + for indices in task_groups + if role_group_counts[assignment[indices[0]]] > 1 + ] + if not movable: + raise LeakageError( + f"Cannot provide evaluator-protected coverage for task {task!r}" + ) + indices = min( + movable, + key=lambda candidate: canonical_hash( + { + "seed": seed, + "protected_task": task, + "sample_ids": sorted( + placeholder[index]["sample_id"] for index in candidate + ), + } + ), + ) + old_role = assignment[indices[0]] + new_role = min( + EVALUATOR_PROTECTED_SPLITS, + key=lambda role: (assigned_counts[role], role), + ) + for index in indices: + assignment[index] = new_role + role_group_counts[old_role] -= 1 + role_group_counts[new_role] += 1 + assigned_counts[old_role] -= len(indices) + assigned_counts[new_role] += len(indices) return [ {**sample, "split": assignment[index]} for index, sample in enumerate(raw_samples) @@ -322,6 +938,7 @@ def leakage_findings( "group_id", "source_family", "temporal_family", + "acquisition_date", "raw_image_sha256", "processed_image_sha256", "label_sha256", @@ -347,6 +964,7 @@ def leakage_findings( "group_id": "S-SPATIAL-GROUP", "source_family": "S-SOURCE-FAMILY", "temporal_family": "S-TEMPORAL-FAMILY", + "acquisition_date": "S-ACQUISITION-DATE", "raw_image_sha256": "S-RAW-IMAGE-DUPLICATE", "processed_image_sha256": "S-PROCESSED-IMAGE-DUPLICATE", "label_sha256": "S-LABEL-DUPLICATE", @@ -429,36 +1047,157 @@ def leakage_findings( return findings +def build_protected_task_coverage( + samples: list[dict[str, Any]], source: dict[str, Any] +) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]: + implemented_tasks = sorted({item["task"] for item in samples}) + exemptions_raw = source.get("protected_task_exemptions") or {} + if not isinstance(exemptions_raw, dict): + raise LeakageError("protected_task_exemptions must be an object") + exemptions: dict[str, str] = {} + for raw_task, raw_reason in exemptions_raw.items(): + task = canonical_identifier(raw_task, field="task", sample_id="") + if task in exemptions: + raise LeakageError(f"Ambiguous protected task exemption for {task!r}") + reason = " ".join(str(raw_reason).split()) + if len(reason) < 20: + raise LeakageError( + f"Protected task exemption for {task!r} requires a concrete reason" + ) + exemptions[task] = reason + unknown = sorted(set(exemptions) - set(implemented_tasks)) + if unknown: + raise LeakageError( + f"Protected task exemptions reference unknown tasks: {unknown}" + ) + coverage: list[dict[str, Any]] = [] + findings: list[dict[str, Any]] = [] + for task in implemented_tasks: + counts = Counter(item["split"] for item in samples if item["task"] == task) + evaluator_count = sum(counts[role] for role in EVALUATOR_PROTECTED_SPLITS) + status = "covered" if evaluator_count else "not_evaluable" + row = { + "task": task, + "status": status, + "test_count": counts["test"], + "background_test_count": counts["background-test"], + "sealed_challenge_count": counts["challenge"], + "exemption_reason": exemptions.get(task), + } + coverage.append(row) + if evaluator_count: + continue + finding = { + "code": ( + "S-PROTECTED-TASK-COVERAGE-EXEMPTED" + if task in exemptions + else "S-PROTECTED-TASK-COVERAGE-MISSING" + ), + "task": task, + "status": "not_evaluable", + } + if task in exemptions: + finding["reason"] = exemptions[task] + findings.append(finding) + return coverage, findings + + +def seal_challenge_sample(sample: dict[str, Any]) -> dict[str, Any]: + sealed_fields = { + "label_sha256", + "label_geometry_hash", + "label_path", + "label_geometry_path", + "label_geometry_fingerprint", + "object_ids", + "native_feature_ids", + "record_sha256", + "governance_binding", + "content_path_bindings", + } + result = {key: value for key, value in sample.items() if key not in sealed_fields} + result["sealed"] = True + result["withheld_fields"] = sorted(sealed_fields) + return result + + +def redact_challenge_findings(findings: list[dict[str, Any]]) -> list[dict[str, Any]]: + sensitive_codes = { + "S-LABEL-DUPLICATE", + "S-LABEL-GEOMETRY-DUPLICATE", + "S-OBJECT-INSTANCE", + "S-NATIVE-FEATURE", + "S-EXACT-DUPLICATE", + } + redacted: list[dict[str, Any]] = [] + for finding in findings: + item = dict(finding) + if item.get("code") in sensitive_codes and "challenge" in item.get( + "splits", [] + ): + item.pop("identity", None) + item["identity_withheld"] = "sealed_challenge_identity" + redacted.append(item) + return redacted + + def build_manifests( source: dict[str, Any], + *, + source_root: Path | None = None, + trusted_fixture_mode: bool = False, ) -> tuple[dict[str, Any], dict[str, Any], dict[str, Any]]: if source.get("schema_version") != 1: raise LeakageError("Unsupported source manifest schema_version") + if ( + not isinstance(source.get("dataset_version"), str) + or not source["dataset_version"].strip() + ): + raise LeakageError("dataset_version must be non-empty") metric_crs = validate_metric_crs(source.get("crs")) - assigned_source_samples = assign_split_roles(source) - samples = [normalized_sample(item) for item in assigned_source_samples] + source_trust = validate_source_trust( + source, source_root, trusted_fixture_mode=trusted_fixture_mode + ) + assigned_source_samples = assign_split_roles( + source, source_root=source_root, source_trust=source_trust + ) + samples = [ + normalized_sample( + item, + source_root=source_root, + source_trust=source_trust, + ) + for item in assigned_source_samples + ] if not samples: raise LeakageError("Source manifest has no samples") if len({item["sample_id"] for item in samples}) != len(samples): - raise LeakageError("Duplicate sample_id in source manifest") - buffer_m = float(source.get("independence_buffer_m") or 0) - if not math.isfinite(buffer_m) or buffer_m <= 0: - raise LeakageError("independence_buffer_m must be a positive finite number") + raise LeakageError( + "Duplicate or ambiguous canonical sample_id in source manifest" + ) + buffer_m, perceptual_threshold, geometry_threshold = validated_policy_thresholds( + source + ) split_counts = Counter(item["split"] for item in samples) - required_splits = set(source.get("required_splits") or NORMATIVE_SPLITS) - unsupported_required = sorted(required_splits - ALL_SPLITS) - if unsupported_required: - raise LeakageError(f"Unsupported required splits: {unsupported_required}") - missing_splits = sorted(required_splits - set(split_counts)) + declared_required = source.get("required_splits") + if ( + declared_required is not None + and tuple(declared_required) != MANDATORY_SPLIT_ROLES + ): + raise LeakageError( + "required_splits cannot change the code-owned mandatory role order" + ) + missing_splits = sorted(NORMATIVE_SPLITS - set(split_counts)) if missing_splits: raise LeakageError(f"Required splits are absent: {missing_splits}") - perceptual_threshold = int(source.get("perceptual_hamming_threshold", 4)) - geometry_threshold = int(source.get("label_geometry_hamming_threshold", 2)) - if not 0 <= perceptual_threshold < 64 or not 0 <= geometry_threshold < 64: - raise LeakageError("Near-duplicate Hamming thresholds must be between 0 and 63") findings = leakage_findings( samples, buffer_m, perceptual_threshold, geometry_threshold ) + protected_task_coverage, coverage_findings = build_protected_task_coverage( + samples, source + ) + findings.extend(coverage_findings) + public_findings = redact_challenge_findings(findings) canonical_source = dict(source) canonical_source["samples"] = sorted( source.get("samples", []), key=lambda item: str(item.get("sample_id", "")) @@ -470,19 +1209,32 @@ def build_manifests( key=lambda item: item["sample_id"], ) protected_samples = sorted( - (item for item in samples if item["split"] in PROTECTED_SPLITS), + ( + seal_challenge_sample(item) if item["split"] == "challenge" else item + for item in samples + if item["split"] in PROTECTED_SPLITS + ), key=lambda item: item["sample_id"], ) + public_source_trust = { + key: value for key, value in source_trust.items() if not key.startswith("_") + } common = { "schema_version": SCHEMA_VERSION, "generator_version": GENERATOR_VERSION, "dataset_version": source["dataset_version"], "source_manifest_sha256": source_hash, + "source_trust": public_source_trust, + "mandatory_split_roles": list(MANDATORY_SPLIT_ROLES), "crs": metric_crs, "independence_buffer_m": buffer_m, "perceptual_hamming_threshold": perceptual_threshold, "label_geometry_hamming_threshold": geometry_threshold, - "assignment_mode": str(source.get("assignment_mode") or "preassigned"), + "assignment_mode": canonical_identifier( + source.get("assignment_mode") or "preassigned", + field="assignment_mode", + sample_id="", + ), "assignment_algorithm": "group-before-split-deficit-balancer-v1", "assignment_seed": str( (source.get("split_assignment") or {}).get("seed") @@ -511,6 +1263,8 @@ def build_manifests( **common, "manifest_role": "protected_release_only", "allowed_splits": sorted(PROTECTED_SPLITS), + "implemented_tasks": sorted({item["task"] for item in samples}), + "protected_task_coverage": protected_task_coverage, "training_access_allowed": False, "selection_use_allowed": False, "labels_available_by_split": { @@ -538,6 +1292,7 @@ def build_manifests( "sample_id", "group_id", "source_family", + "acquisition_date", "temporal_family", "object_ids", "native_feature_ids", @@ -557,11 +1312,202 @@ def build_manifests( "perceptual_hamming_threshold": perceptual_threshold, "label_geometry_hamming_threshold": geometry_threshold, "finding_count": len(findings), - "findings": findings, + "findings": public_findings, } return development, protected, leakage +def validate_protected_manifest( + protected_manifest: dict[str, Any], + *, + trusted_fixture_mode: bool = False, +) -> tuple[dict[str, dict[str, Any]], dict[str, dict[str, Any]]] | None: + if not isinstance(protected_manifest, dict) or not protected_manifest: + raise LeakageError("Protected manifest trust validation failed: empty manifest") + required = { + "schema_version", + "generator_version", + "dataset_version", + "source_manifest_sha256", + "source_trust", + "mandatory_split_roles", + "manifest_role", + "allowed_splits", + "implemented_tasks", + "protected_task_coverage", + "samples", + "manifest_sha256", + } + missing = sorted(required - set(protected_manifest)) + if missing: + raise LeakageError( + f"Protected manifest trust validation failed: missing fields {missing}" + ) + if protected_manifest["schema_version"] != SCHEMA_VERSION: + raise LeakageError("Protected manifest trust validation failed: schema_version") + if protected_manifest["generator_version"] != GENERATOR_VERSION: + raise LeakageError( + "Protected manifest trust validation failed: generator_version" + ) + if protected_manifest["manifest_role"] != "protected_release_only": + raise LeakageError("Protected manifest trust validation failed: manifest_role") + if tuple(protected_manifest["mandatory_split_roles"]) != MANDATORY_SPLIT_ROLES: + raise LeakageError( + "Protected manifest trust validation failed: mandatory roles" + ) + if set(protected_manifest["allowed_splits"]) != PROTECTED_SPLITS: + raise LeakageError("Protected manifest trust validation failed: allowed splits") + require_hex( + "", + "source_manifest_sha256", + protected_manifest["source_manifest_sha256"], + SHA256_PATTERN, + ) + expected_manifest_hash = require_hex( + "", + "manifest_sha256", + protected_manifest["manifest_sha256"], + SHA256_PATTERN, + ) + unsigned = { + key: value + for key, value in protected_manifest.items() + if key != "manifest_sha256" + } + if canonical_hash(unsigned) != expected_manifest_hash: + raise LeakageError( + "Protected manifest trust validation failed: checksum mismatch" + ) + trust = protected_manifest["source_trust"] + if not isinstance(trust, dict): + raise LeakageError("Protected manifest trust validation failed: source trust") + evidence_indexes = None + if trust.get("mode") == "synthetic_fixture": + if not trusted_fixture_mode: + raise LeakageError( + "Protected synthetic fixture requires explicit trusted_fixture_mode=True" + ) + if ( + protected_manifest["dataset_version"] + != TRUSTED_FIXTURE_POLICY["dataset_version"] + or protected_manifest.get("claim_boundary") + != TRUSTED_FIXTURE_POLICY["claim_boundary"] + or trust.get("policy_id") != TRUSTED_FIXTURE_POLICY["policy_id"] + or trust.get("production_accuracy_use_allowed") is not False + ): + raise LeakageError( + "Protected manifest trust validation failed: fixture policy binding" + ) + elif trust.get("mode") == "governed_production": + evidence = trust.get("verified_evidence") + if ( + trust.get("production_accuracy_use_allowed") is not True + or not isinstance(evidence, dict) + or set(evidence) + != { + "p3_scan_manifest", + "source_provenance_manifest", + } + ): + raise LeakageError( + "Protected manifest trust validation failed: provenance binding" + ) + lean_evidence: dict[str, dict[str, Any]] = {} + for key, binding in evidence.items(): + if not isinstance(binding, dict): + raise LeakageError( + "Protected manifest trust validation failed: provenance binding" + ) + expected_fields = ( + {"path", "resolved_path", "sha256", "scan_id", "scanner_version"} + if key == "p3_scan_manifest" + else { + "path", + "resolved_path", + "sha256", + "manifest_type", + "records_canonical_json_sha256", + } + ) + if set(binding) != expected_fields: + raise LeakageError( + "Protected manifest trust validation failed: evidence schema" + ) + lean_evidence[key] = { + field: binding[field] for field in ("path", "resolved_path", "sha256") + } + verified, p3_items, provenance_records = _validated_governance_bundle( + lean_evidence, source_root=None, from_protected_manifest=True + ) + if ( + verified["p3_scan_manifest"]["scan_id"] + != evidence["p3_scan_manifest"]["scan_id"] + or verified["p3_scan_manifest"]["scanner_version"] + != evidence["p3_scan_manifest"]["scanner_version"] + or verified["source_provenance_manifest"]["manifest_type"] + != evidence["source_provenance_manifest"]["manifest_type"] + or verified["source_provenance_manifest"]["records_canonical_json_sha256"] + != evidence["source_provenance_manifest"]["records_canonical_json_sha256"] + ): + raise LeakageError( + "Protected manifest trust validation failed: evidence attestation" + ) + evidence_indexes = (p3_items, provenance_records) + else: + raise LeakageError( + "Protected manifest trust validation failed: unsupported trust mode" + ) + samples = protected_manifest["samples"] + if not isinstance(samples, list) or not samples: + raise LeakageError("Protected manifest trust validation failed: empty samples") + split_counts = Counter( + item.get("split") for item in samples if isinstance(item, dict) + ) + if set(split_counts) != PROTECTED_SPLITS or any( + split_counts[role] < 1 for role in PROTECTED_SPLITS + ): + raise LeakageError( + "Protected manifest trust validation failed: protected coverage" + ) + for item in samples: + if not isinstance(item, dict): + raise LeakageError( + "Protected manifest trust validation failed: sample type" + ) + sample_id = canonical_identifier( + item.get("sample_id"), field="sample_id", sample_id="" + ) + if sample_id != item["sample_id"]: + raise LeakageError( + "Protected manifest trust validation failed: noncanonical sample" + ) + if item["split"] == "challenge": + forbidden = { + "label_sha256", + "label_geometry_hash", + "label_geometry_fingerprint", + "object_ids", + "native_feature_ids", + "record_sha256", + "governance_binding", + "content_path_bindings", + } + if forbidden & set(item) or item.get("sealed") is not True: + raise LeakageError( + "Protected manifest trust validation failed: challenge seal" + ) + coverage = protected_manifest["protected_task_coverage"] + if not isinstance(coverage, list) or { + row.get("task") for row in coverage if isinstance(row, dict) + } != set(protected_manifest["implemented_tasks"]): + raise LeakageError("Protected manifest trust validation failed: task coverage") + if any(row.get("status") != "covered" for row in coverage): + raise LeakageError( + "Protected manifest trust validation failed: unevaluable task" + ) + return evidence_indexes + + def protected_identities(protected_manifest: dict[str, Any]) -> set[str]: identities: set[str] = set() scalar_fields = ( @@ -570,6 +1516,7 @@ def protected_identities(protected_manifest: dict[str, Any]) -> set[str]: "record_sha256", "source_family", "temporal_family", + "acquisition_date", "raw_image_sha256", "processed_image_sha256", "label_sha256", @@ -580,7 +1527,11 @@ def protected_identities(protected_manifest: dict[str, Any]) -> set[str]: "acquisition_id", ) for item in protected_manifest.get("samples", []): - identities.update(str(item[field]) for field in scalar_fields) + identities.update( + str(item[field]) + for field in scalar_fields + if field in item and str(item[field]) + ) identities.update(str(value) for value in item.get("object_ids", [])) identities.update(str(value) for value in item.get("native_feature_ids", [])) return identities @@ -590,23 +1541,81 @@ def assert_training_inputs_safe( input_paths: Iterable[Path], input_records: Iterable[dict[str, Any]], protected_manifest: dict[str, Any], + *, + trusted_fixture_mode: bool = False, ) -> None: - """Refuse non-train roles and protected identities at a fitting boundary.""" + """Refuse untrusted manifests, non-train roles and protected identities.""" + evidence_indexes = validate_protected_manifest( + protected_manifest, trusted_fixture_mode=trusted_fixture_mode + ) + production_mode = ( + protected_manifest["source_trust"]["mode"] == "governed_production" + ) forbidden = protected_identities(protected_manifest) violations: list[str] = [] - for path in input_paths: - lowered = path.as_posix().lower() + paths = [Path(path) for path in input_paths] + records = list(input_records) + if not paths and not records: + violations.append("empty_training_inputs") + forbidden_content_hashes = { + value for value in forbidden if SHA256_PATTERN.fullmatch(value) + } + for path in paths: + lowered = unicodedata.normalize("NFKC", path.as_posix()).casefold() if any( token in lowered for token in ("protected", "holdout", "challenge", "background-test") ): violations.append(f"protected_path:{path}") + if not path.is_file(): + violations.append(f"unverifiable_training_path:{path}") + continue + content_hash = sha256_file(path) + if content_hash in forbidden_content_hashes: + violations.append(f"protected_content_hash:{path}") + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except (OSError, UnicodeDecodeError, json.JSONDecodeError): + payload = None + if isinstance(payload, dict): + if payload.get("manifest_role") == "protected_release_only": + violations.append(f"protected_manifest_content:{path}") + + def collect_splits(value: Any) -> set[str]: + if isinstance(value, dict): + found = ( + { + canonical_identifier( + value["split"], + field="split", + sample_id="", + ) + } + if "split" in value and isinstance(value["split"], str) + else set() + ) + for nested in value.values(): + found.update(collect_splits(nested)) + return found + if isinstance(value, list): + found: set[str] = set() + for nested in value: + found.update(collect_splits(nested)) + return found + return set() + + non_train = sorted(collect_splits(payload) - TRAIN_SPLITS) + if non_train: + violations.append(f"non_train_content:{path}:{','.join(non_train)}") + if paths and not records: + violations.append("unbound_training_paths") scalar_fields = ( "sample_id", "group_id", "record_sha256", "source_family", "temporal_family", + "acquisition_date", "raw_image_sha256", "processed_image_sha256", "label_sha256", @@ -616,14 +1625,179 @@ def assert_training_inputs_safe( "parent_raster_id", "acquisition_id", ) - for record in input_records: - if record.get("split") not in TRAIN_SPLITS: - violations.append( - f"non_train_role:{record.get('sample_id')}:{record.get('split')}" + required_record_fields = { + "sample_id", + "split", + "group_id", + "source_family", + "temporal_family", + "acquisition_date", + "raw_image_sha256", + "processed_image_sha256", + "label_sha256", + "label_geometry_hash", + "perceptual_image_hash", + "label_geometry_fingerprint", + "parent_raster_id", + "acquisition_id", + "object_ids", + "native_feature_ids", + } + canonical_scalar_identifiers = { + "sample_id", + "group_id", + "source_family", + "temporal_family", + "parent_raster_id", + "acquisition_id", + } + + def production_binding_violation(record: dict[str, Any]) -> str | None: + if not production_mode: + return None + if evidence_indexes is None: + return "production_evidence_unavailable" + p3_items, provenance_records = evidence_indexes + sample_id = str(record.get("sample_id", "")) + try: + governance = record.get("governance_binding") + governance_fields = {"source_provenance_record_id", "p3_item_ids"} + if not isinstance(governance, dict) or set(governance) != governance_fields: + return f"missing_governance_binding:{sample_id}" + provenance_id = _required_string( + governance["source_provenance_record_id"], + field=f"{sample_id}.source_provenance_record_id", ) - values = {str(record.get(field, "")) for field in scalar_fields} - values.update(str(value) for value in record.get("object_ids", [])) - values.update(str(value) for value in record.get("native_feature_ids", [])) + p3_ids = governance["p3_item_ids"] + if not isinstance(p3_ids, dict) or set(p3_ids) != set(ASSET_BINDINGS): + return f"incomplete_p3_binding:{sample_id}" + provenance = provenance_records.get(provenance_id) + if provenance is None or provenance["sample_id"] != sample_id: + return f"unavailable_provenance_record:{sample_id}" + for field in ("perceptual_image_hash", "label_geometry_fingerprint"): + if record.get(field) != provenance[field]: + return f"provenance_fingerprint_mismatch:{sample_id}:{field}" + content = record.get("content_path_bindings") + if not isinstance(content, dict) or set(content) != set(ASSET_BINDINGS): + return f"missing_accessible_content_paths:{sample_id}" + binding_fields = { + "path", + "resolved_path", + "sha256", + "size_bytes", + "p3_item_id", + "source_provenance_record_id", + } + for role, (path_field, hash_field) in ASSET_BINDINGS.items(): + bound = content[role] + if not isinstance(bound, dict) or set(bound) != binding_fields: + return f"invalid_content_binding:{sample_id}:{role}" + path = _resolve_evidence_path(bound["resolved_path"], None) + if not path.is_file(): + return f"unavailable_content_path:{sample_id}:{role}" + actual = sha256_file(path) + size = path.stat().st_size + p3_id = require_hex( + sample_id, + f"content_path_bindings.{role}.p3_item_id", + bound["p3_item_id"], + P3_ITEM_ID_PATTERN, + ) + asset = provenance["assets"][role] + p3_item = p3_items.get(p3_id) + canonical_path = _canonical_bound_path( + bound["path"], field=f"{sample_id}.{path_field}" + ) + if ( + record.get(path_field) != bound["path"] + or record.get(hash_field) != actual + or bound["sha256"] != actual + or bound["size_bytes"] != size + or bound["source_provenance_record_id"] != provenance_id + or p3_ids[role] != p3_id + or asset["path"] != canonical_path + or asset["sha256"] != actual + or asset["size_bytes"] != size + or asset["p3_item_id"] != p3_id + or p3_item is None + ): + return f"record_path_hash_binding_mismatch:{sample_id}:{role}" + if ( + _canonical_bound_path(p3_item["path"], field=f"P3 {p3_id}.path") + != canonical_path + or p3_item["sha256"] != actual + or p3_item["size_bytes"] != size + or p3_item["status"] != "examined" + or p3_item["read_status"] != "readable" + or p3_item["recommended_action"] != "accept" + or p3_item["empty_content"] is not False + or p3_item["schema_conformity"] + not in {"conformant", "not_applicable"} + or p3_item["anomalies"] + ): + return f"unaccepted_p3_binding:{sample_id}:{role}" + unsigned_record = { + key: value + for key, value in record.items() + if key not in {"split", "record_sha256"} + } + if canonical_hash(unsigned_record) != record.get("record_sha256"): + return f"record_checksum_mismatch:{sample_id}" + except (KeyError, OSError, TypeError, LeakageError) as exc: + return f"unverifiable_production_record:{sample_id}:{exc}" + return None + + for record in records: + binding_violation = production_binding_violation(record) + if binding_violation: + violations.append(binding_violation) + missing = sorted(required_record_fields - set(record)) + if missing: + violations.append( + f"incomplete_training_record:{record.get('sample_id')}:{','.join(missing)}" + ) + raw_split = record.get("split") + try: + split = canonical_identifier( + raw_split, field="split", sample_id=str(record.get("sample_id")) + ) + except LeakageError: + split = "" + if split not in TRAIN_SPLITS: + violations.append(f"non_train_role:{record.get('sample_id')}:{raw_split}") + values: set[str] = set() + for field in scalar_fields: + value = record.get(field) + if value in (None, ""): + continue + if field in canonical_scalar_identifiers: + try: + value = canonical_identifier( + value, field=field, sample_id=str(record.get("sample_id")) + ) + except LeakageError: + violations.append(f"invalid_training_identity:{field}") + continue + elif field in {"acquisition_date"}: + try: + value = date.fromisoformat(str(value)).isoformat() + except ValueError: + violations.append("invalid_training_identity:acquisition_date") + continue + else: + value = str(value).lower() + values.add(str(value)) + for field in ("object_ids", "native_feature_ids"): + try: + values.update( + normalized_identifier_list( + record.get(field, []), + field=field, + sample_id=str(record.get("sample_id")), + ) + ) + except LeakageError: + violations.append(f"invalid_training_identity:{field}") overlap = sorted((values - {""}) & forbidden) if overlap: violations.append(f"protected_identity:{','.join(overlap)}") @@ -633,32 +1807,123 @@ def assert_training_inputs_safe( ) -def generate(source_path: Path, output_dir: Path) -> dict[str, Any]: - source = json.loads(source_path.read_text(encoding="utf-8")) - development, protected, leakage = build_manifests(source) +def invalidate_consumable_outputs( + output_dir: Path, + *, + error: str, + source_manifest_sha256: str | None, + leakage: dict[str, Any] | None = None, +) -> dict[str, Any]: output_dir.mkdir(parents=True, exist_ok=True) + failure_id = canonical_hash( + { + "generator_version": GENERATOR_VERSION, + "source_manifest_sha256": source_manifest_sha256, + "error": error, + } + ) + status = { + "schema_version": SCHEMA_VERSION, + "generator_version": GENERATOR_VERSION, + "status": "fail", + "failure_id": failure_id, + "error": error, + "source_manifest_sha256": source_manifest_sha256, + "development_manifest_sha256": None, + "protected_manifest_sha256": None, + "consumable_manifests_valid": False, + } + tombstone = { + "schema_version": SCHEMA_VERSION, + "generator_version": GENERATOR_VERSION, + "status": "invalidated", + "failure_id": failure_id, + "consumable": False, + "reason": "Latest regeneration failed; stale split content is fail-closed.", + } + failure_report = leakage or { + "schema_version": SCHEMA_VERSION, + "generator_version": GENERATOR_VERSION, + "status": "fail", + "source_manifest_sha256": source_manifest_sha256, + "finding_count": 1, + "findings": [ + { + "code": "S-GENERATION-ERROR", + "message": error, + } + ], + } + write_json(output_dir / "generation-status.json", status) + write_json(output_dir / "leakage-gate-report.json", failure_report) + write_json(output_dir / "development-split-manifest.json", tombstone) + write_json(output_dir / "protected-split-manifest.json", tombstone) + return status + + +def generate( + source_path: Path, + output_dir: Path, + *, + trusted_fixture_mode: bool = False, +) -> dict[str, Any]: + output_dir.mkdir(parents=True, exist_ok=True) + source_file_hash: str | None = None + try: + source_file_hash = sha256_file(source_path) + source = json.loads(source_path.read_text(encoding="utf-8")) + development, protected, leakage = build_manifests( + source, + source_root=source_path.parent, + trusted_fixture_mode=trusted_fixture_mode, + ) + except (OSError, json.JSONDecodeError, LeakageError) as exc: + invalidate_consumable_outputs( + output_dir, + error=str(exc), + source_manifest_sha256=source_file_hash, + ) + raise generation_status = { "schema_version": SCHEMA_VERSION, "generator_version": GENERATOR_VERSION, "status": leakage["status"], "source_manifest_sha256": leakage["source_manifest_sha256"], + "source_file_sha256": source_file_hash, "development_manifest_sha256": ( development["manifest_sha256"] if leakage["status"] == "pass" else None ), "protected_manifest_sha256": ( protected["manifest_sha256"] if leakage["status"] == "pass" else None ), + "consumable_manifests_valid": leakage["status"] == "pass", } - write_json(output_dir / "generation-status.json", generation_status) - write_json(output_dir / "leakage-gate-report.json", leakage) if leakage["status"] != "pass": - raise LeakageError( - f"Leakage gate failed with {leakage['finding_count']} findings" + error = f"Leakage gate failed with {leakage['finding_count']} findings" + invalidate_consumable_outputs( + output_dir, + error=error, + source_manifest_sha256=leakage["source_manifest_sha256"], + leakage=leakage, ) + raise LeakageError(error) train_samples = [ item for item in development["samples"] if item["split"] == "train" ] - assert_training_inputs_safe([], train_samples, protected) + try: + assert_training_inputs_safe( + [], train_samples, protected, trusted_fixture_mode=trusted_fixture_mode + ) + except LeakageError as exc: + invalidate_consumable_outputs( + output_dir, + error=str(exc), + source_manifest_sha256=leakage["source_manifest_sha256"], + leakage=leakage, + ) + raise + write_json(output_dir / "generation-status.json", generation_status) + write_json(output_dir / "leakage-gate-report.json", leakage) write_json(output_dir / "development-split-manifest.json", development) write_json(output_dir / "protected-split-manifest.json", protected) return { diff --git a/scripts/run_accuracy_phase4_benchmark.py b/scripts/run_accuracy_phase4_benchmark.py index 41c3a7e5..43004850 100644 --- a/scripts/run_accuracy_phase4_benchmark.py +++ b/scripts/run_accuracy_phase4_benchmark.py @@ -6,12 +6,18 @@ from __future__ import annotations import argparse import hashlib import json +import math +import os import platform +import re +import shutil import subprocess import sys +import tempfile +from datetime import datetime from importlib import metadata as importlib_metadata from pathlib import Path -from typing import Any +from typing import Any, Callable ROOT = Path(__file__).resolve().parents[1] BACKEND_ROOT = ROOT / "backend" @@ -19,7 +25,15 @@ for entry in (str(ROOT), str(BACKEND_ROOT), str(ROOT / "scripts")): if entry not in sys.path: sys.path.insert(0, entry) -from accuracy_phase4_evaluator import EVALUATOR_VERSION, canonical_hash, evaluate_cases # noqa: E402 +from accuracy_phase4_evaluator import ( # noqa: E402 + EVALUATOR_VERSION, + EXPECTED_PROTECTED_POLICY, + SUBGROUP_MIN_CASE_SUPPORT, + TASKS, + canonical_hash, + evaluate_cases, + task_inventory, +) from generate_accuracy_phase4_splits import ( # noqa: E402 GENERATOR_VERSION, LeakageError, @@ -31,6 +45,90 @@ from run_golden_qa_benchmark import run_benchmark # noqa: E402 WORKFLOW_VERSION = "2.0.0" BENCHMARK_ID = "geointel-p4-reference-harness-v2" GATE_STATES = {"pass", "fail", "not_evaluable"} +PRODUCT_BASELINE_SCHEMA_VERSION = 2 +PRODUCT_BASELINE_MANIFEST_TYPE = "geointel_governed_product_baseline" +PRODUCT_ARTIFACT_ROLES = ( + "configuration", + "protected_split_manifest", + "authoritative_reference_manifest", + "inference_evidence", + "raw_predictions", + "metric_report", + "human_review_ledger", + "geometric_leakage_audit", + "vault_access_evidence", +) +PROTECTED_SPLIT_ROLES = {"test", "background-test", "challenge"} +EVALUATED_PROTECTED_SPLIT_ROLES = {"test", "background-test"} +REQUIRED_SUBGROUP_DIMENSION_FIELDS = { + "region": "region", + "municipality": "municipality", + "urbanity": "urbanity", + "object_size": "object_size", + "source": "source", + "sensor": "sensor", + "resolution": "resolution_m", + "season": "season", + "date": "date", + "vegetation": "vegetation", + "occlusion": "occlusion", + "difficulty": "difficulty", + "context": "context", +} +REQUIRED_SUBGROUP_DIMENSIONS = frozenset(REQUIRED_SUBGROUP_DIMENSION_FIELDS) +SUBGROUP_RELEASE_POLICY = { + "minimum_case_support_per_task_stratum": SUBGROUP_MIN_CASE_SUPPORT, + "minimum_distinct_strata_per_dimension": 2, + "required_dimensions": sorted(REQUIRED_SUBGROUP_DIMENSIONS), + "missing_or_insufficient_support": "fail", + "targets_must_be_frozen_before_protected_access": True, +} +REQUIRED_EVALUATOR_TASK_FAMILIES = frozenset(TASKS) +SHA256_PATTERN = re.compile(r"^[0-9a-f]{64}$") + +LOCAL_GATE_NAMES = frozenset( + { + "all_declared_evaluator_families_exercised", + "implemented_capability_inventory", + "normative_split_roles_and_leakage", + "manifest_training_firewall_contract", + "protected_operating_point_contract", + "complete_raw_predictions_retained", + "reference_implementation_baseline", + "stratified_metric_contract", + "undefined_metric_truth_table", + } +) +PRODUCT_GATE_NAMES = frozenset( + { + "active_model_available_and_hash_verified", + "authoritative_reference_portfolio_available", + "human_review_complete", + "split_independence", + "phase3_leakage_resolved", + "protected_storage_isolation", + "executed_product_incumbent_baseline", + "representative_product_subgroup_support", + } +) +REQUIRED_AUTHORITY_REQUIREMENTS = ( + {"task": "building_validation", "zone": "flanders", "primary": "grb"}, + {"task": "building_validation", "zone": "wallonia", "primary": "picc"}, + {"task": "building_validation", "zone": "brussels", "primary": "urbis"}, + {"task": "terrain_height", "zone": "flanders", "primary": "dhmv"}, + {"task": "terrain_height", "zone": "wallonia", "primary": "spw_terrain"}, + { + "task": "north_sea_bathymetry", + "zone": "belgian_north_sea", + "primary": "mdk", + }, + { + "task": "imagery_corroboration", + "zone": "belgium", + "primary": "official_orthophoto", + "contextual": "sentinel-2", + }, +) class EvidenceConflictError(RuntimeError): @@ -51,18 +149,106 @@ def json_bytes(payload: Any) -> bytes: ).encode("utf-8") +def is_sha256(value: Any) -> bool: + return isinstance(value, str) and SHA256_PATTERN.fullmatch(value) is not None + + +def parse_timestamp(value: Any) -> datetime | None: + if not isinstance(value, str) or not value.strip(): + return None + try: + parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) + except ValueError: + return None + return parsed if parsed.tzinfo is not None else None + + +def _raise_evidence_conflict(path: Path, detail: str = "different content") -> None: + raise EvidenceConflictError( + f"Refusing to overwrite immutable evidence with {detail}: {path}" + ) + + def write_json_immutable(path: Path, payload: Any) -> None: + """Create one immutable JSON file without a check-then-overwrite race.""" + content = json_bytes(payload) path.parent.mkdir(parents=True, exist_ok=True) if path.exists(): - if path.read_bytes() != content: - raise EvidenceConflictError( - f"Refusing to overwrite immutable evidence with different content: {path}" - ) + if not path.is_file() or path.read_bytes() != content: + _raise_evidence_conflict(path) return - temporary = path.with_name(f".{path.name}.tmp") - temporary.write_bytes(content) - temporary.replace(path) + file_descriptor, temporary_name = tempfile.mkstemp( + prefix=f".{path.name}.", + suffix=".tmp", + dir=path.parent, + ) + temporary = Path(temporary_name) + try: + with os.fdopen(file_descriptor, "wb") as stream: + stream.write(content) + stream.flush() + os.fsync(stream.fileno()) + try: + os.link(temporary, path) + except FileExistsError: + if not path.is_file() or path.read_bytes() != content: + _raise_evidence_conflict(path) + finally: + temporary.unlink(missing_ok=True) + + +def _assert_immutable_bundle( + output_dir: Path, + contents: dict[str, bytes], +) -> None: + if not output_dir.is_dir(): + _raise_evidence_conflict(output_dir, "a non-directory target") + observed_entries = { + path.relative_to(output_dir).as_posix(): ( + "file" if path.is_file() else "directory" if path.is_dir() else "other" + ) + for path in output_dir.rglob("*") + } + expected_entries = {name: "file" for name in contents} + if observed_entries != expected_entries: + _raise_evidence_conflict( + output_dir, + "an incomplete, nested or unexpected artifact set; " + f"expected={sorted(expected_entries.items())}, " + f"observed={sorted(observed_entries.items())}", + ) + for name, content in contents.items(): + path = output_dir / name + if not path.is_file() or path.read_bytes() != content: + _raise_evidence_conflict(path) + + +def write_json_bundle_immutable(output_dir: Path, payloads: dict[str, Any]) -> None: + """Publish a complete immutable evidence bundle with an atomic directory rename.""" + + contents = {name: json_bytes(payload) for name, payload in payloads.items()} + output_dir.parent.mkdir(parents=True, exist_ok=True) + if output_dir.exists(): + _assert_immutable_bundle(output_dir, contents) + return + staging = Path( + tempfile.mkdtemp(prefix=f".{output_dir.name}.", dir=output_dir.parent) + ) + try: + for name, content in sorted(contents.items()): + path = staging / name + with path.open("xb") as stream: + stream.write(content) + stream.flush() + os.fsync(stream.fileno()) + try: + staging.replace(output_dir) + except FileExistsError: + _assert_immutable_bundle(output_dir, contents) + finally: + if staging.exists(): + shutil.rmtree(staging) def repository_commit(repo_root: Path) -> str | None: @@ -113,103 +299,1849 @@ def canonical_golden_baseline() -> dict[str, Any]: } +def _canonical_id_hash(values: set[str]) -> str: + return canonical_hash(sorted(values)) + + +def _load_governed_artifacts( + repo_root: Path, + governed_root: Path, + manifest: dict[str, Any], + violations: list[str], +) -> tuple[dict[str, dict[str, Any]], list[dict[str, Any]]]: + payloads: dict[str, dict[str, Any]] = {} + checked: list[dict[str, Any]] = [] + observed_paths: set[Path] = set() + for role in PRODUCT_ARTIFACT_ROLES: + descriptor = manifest.get(role) + if not isinstance(descriptor, dict): + violations.append(f"{role}:descriptor_not_object") + continue + descriptor_fields = {"path", "sha256", "size_bytes"} + missing_fields = descriptor_fields - set(descriptor) + unexpected_fields = set(descriptor) - descriptor_fields + if missing_fields: + violations.append( + f"{role}:descriptor_missing:{','.join(sorted(missing_fields))}" + ) + if unexpected_fields: + violations.append( + f"{role}:descriptor_unexpected:{','.join(sorted(unexpected_fields))}" + ) + relative_path = descriptor.get("path") + if ( + not isinstance(relative_path, str) + or not relative_path.strip() + or Path(relative_path).is_absolute() + ): + violations.append(f"{role}:path_not_relative") + continue + artifact_path = (repo_root / relative_path).resolve() + try: + artifact_path.relative_to(governed_root.resolve()) + except (OSError, ValueError): + violations.append(f"{role}:outside_governed_evidence_root") + continue + if artifact_path in observed_paths: + violations.append(f"{role}:artifact_path_reused") + continue + observed_paths.add(artifact_path) + if not artifact_path.is_file(): + violations.append(f"{role}:missing") + continue + observed_hash = sha256(artifact_path) + observed_size = artifact_path.stat().st_size + checked.append( + { + "role": role, + "path": artifact_path.relative_to(repo_root.resolve()).as_posix(), + "sha256": observed_hash, + "size_bytes": observed_size, + } + ) + if not is_sha256(descriptor.get("sha256")): + violations.append(f"{role}:invalid_declared_sha256") + elif observed_hash != descriptor["sha256"]: + violations.append(f"{role}:hash_mismatch") + if descriptor.get("size_bytes") != observed_size: + violations.append(f"{role}:size_mismatch") + try: + payload = json.loads(artifact_path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + violations.append(f"{role}:invalid_json:{type(exc).__name__}") + continue + if not isinstance(payload, dict): + violations.append(f"{role}:payload_not_object") + continue + payloads[role] = payload + return payloads, checked + + +def _validate_product_manifest_contract( + manifest: dict[str, Any], + active_model: dict[str, Any], + evaluator_hash: str, +) -> list[str]: + violations: list[str] = [] + required_fields = { + "schema_version", + "manifest_type", + "baseline_id", + "created_at", + "status", + "synthetic", + "active_model", + "active_model_sha256", + "evaluator_sha256", + "configuration_sha256", + "development_split_manifest_sha256", + "selection_isolation", + "inference", + *PRODUCT_ARTIFACT_ROLES, + } + missing_fields = required_fields - set(manifest) + unexpected_fields = set(manifest) - required_fields + if missing_fields: + violations.append(f"manifest:missing:{','.join(sorted(missing_fields))}") + if unexpected_fields: + violations.append(f"manifest:unexpected:{','.join(sorted(unexpected_fields))}") + if manifest.get("schema_version") != PRODUCT_BASELINE_SCHEMA_VERSION: + violations.append("manifest:schema_version") + if manifest.get("manifest_type") != PRODUCT_BASELINE_MANIFEST_TYPE: + violations.append("manifest:type") + if ( + not isinstance(manifest.get("baseline_id"), str) + or not manifest["baseline_id"].strip() + ): + violations.append("manifest:baseline_id") + if parse_timestamp(manifest.get("created_at")) is None: + violations.append("manifest:created_at") + if manifest.get("status") != "pass": + violations.append("manifest:status_not_pass") + if manifest.get("synthetic") is not False: + violations.append("manifest:synthetic_or_unspecified") + + expected_identity = { + key: active_model.get(key) + for key in ("model_id", "model_version", "sha256", "size_bytes") + } + if manifest.get("active_model") != expected_identity: + violations.append("manifest:active_model_identity_mismatch") + expected_model_hash = active_model.get("sha256") + if not is_sha256(expected_model_hash): + violations.append("manifest:configured_active_model_sha256_invalid") + if manifest.get("active_model_sha256") != expected_model_hash: + violations.append("manifest:active_model_hash_mismatch") + if manifest.get("evaluator_sha256") != evaluator_hash: + violations.append("manifest:evaluator_hash_mismatch") + for field in ("configuration_sha256", "development_split_manifest_sha256"): + if not is_sha256(manifest.get(field)): + violations.append(f"manifest:{field}_invalid") + + inference = manifest.get("inference") + if not isinstance(inference, dict): + violations.append("manifest:inference_not_object") + else: + if inference.get("executed") is not True: + violations.append("manifest:inference_not_executed") + if inference.get("test_used_for_selection") is not False: + violations.append("manifest:protected_test_selection_policy_invalid") + if re.fullmatch(r"cuda:\d+", str(inference.get("device") or "")) is None: + violations.append("manifest:cuda_device_invalid") + if ( + not isinstance(inference.get("execution_id"), str) + or not inference["execution_id"].strip() + ): + violations.append("manifest:execution_id") + + isolation = manifest.get("selection_isolation") + required_isolation = { + "test_used_for_training": False, + "test_used_for_threshold_selection": False, + "test_used_for_model_selection": False, + "test_used_for_iterative_error_correction": False, + "challenge_labels_accessed": False, + "operating_point_frozen_before_protected_inference": True, + "configuration_sha256": manifest.get("configuration_sha256"), + } + if isolation != required_isolation: + violations.append("manifest:selection_isolation_invalid") + return violations + + +def _validate_configuration( + payload: dict[str, Any], + manifest: dict[str, Any], +) -> list[str]: + violations: list[str] = [] + if payload.get("schema_version") != 2: + violations.append("configuration:schema_version") + if payload.get("artifact_role") != "frozen_inference_configuration": + violations.append("configuration:artifact_role") + if payload.get("active_model_sha256") != manifest.get("active_model_sha256"): + violations.append("configuration:active_model_hash_mismatch") + if payload.get("development_split_manifest_sha256") != manifest.get( + "development_split_manifest_sha256" + ): + violations.append("configuration:development_split_hash_mismatch") + if payload.get("frozen_before_protected_access") is not True: + violations.append("configuration:not_frozen_before_protected_access") + if parse_timestamp(payload.get("frozen_at")) is None: + violations.append("configuration:frozen_at") + if payload.get("protected_data_used") is not False: + violations.append("configuration:protected_data_used") + if payload.get("threshold_selection_source") not in { + "validation_only", + "calibration_only", + "validation_and_calibration", + }: + violations.append("configuration:threshold_selection_source") + parameters = payload.get("parameters_by_task") + if ( + not isinstance(parameters, dict) + or set(parameters) != REQUIRED_EVALUATOR_TASK_FAMILIES + ): + violations.append("configuration:parameters_by_task") + elif any(not isinstance(value, dict) for value in parameters.values()): + violations.append("configuration:task_parameters_not_objects") + if payload.get("subgroup_release_policy") != SUBGROUP_RELEASE_POLICY: + violations.append("configuration:subgroup_release_policy") + targets = payload.get("subgroup_release_targets") + if ( + not isinstance(targets, dict) + or set(targets) != REQUIRED_EVALUATOR_TASK_FAMILIES + ): + violations.append("configuration:subgroup_release_targets") + elif payload.get("subgroup_release_targets_sha256") != canonical_hash(targets): + violations.append("configuration:subgroup_release_targets_sha256") + return violations + + +def _validate_protected_split( + payload: dict[str, Any], +) -> tuple[list[str], dict[str, dict[str, Any]]]: + violations: list[str] = [] + samples_by_id: dict[str, dict[str, Any]] = {} + if payload.get("schema_version") != 3: + violations.append("protected_split:schema_version") + if payload.get("artifact_role") != "protected_evaluation_split": + violations.append("protected_split:artifact_role") + expected_policy = { + "immutable": True, + "training_allowed": False, + "threshold_selection_allowed": False, + "model_selection_allowed": False, + "iterative_error_correction_allowed": False, + "challenge_labels_accessible": False, + } + if payload.get("protected_policy") != expected_policy: + violations.append("protected_split:policy") + if payload.get("evaluator_task_inventory_sha256") != canonical_hash( + task_inventory() + ): + violations.append("protected_split:evaluator_task_inventory_sha256") + + samples = payload.get("samples") + if not isinstance(samples, list) or not samples: + violations.append("protected_split:samples") + return violations, samples_by_id + split_counts: dict[str, int] = {} + for index, sample in enumerate(samples): + prefix = f"protected_split:sample:{index}" + if not isinstance(sample, dict): + violations.append(f"{prefix}:not_object") + continue + sample_id = sample.get("sample_id") + if ( + not isinstance(sample_id, str) + or not sample_id + or sample_id != sample_id.strip() + ): + violations.append(f"{prefix}:sample_id") + continue + if sample_id in samples_by_id: + violations.append(f"{prefix}:duplicate_sample_id") + continue + samples_by_id[sample_id] = sample + split = sample.get("split") + if split not in PROTECTED_SPLIT_ROLES: + violations.append(f"{prefix}:split") + else: + split_counts[split] = split_counts.get(split, 0) + 1 + if sample.get("task") not in REQUIRED_EVALUATOR_TASK_FAMILIES: + violations.append(f"{prefix}:task") + for field in ("zone", "aoi_id"): + value = sample.get(field) + if not isinstance(value, str) or not value.strip(): + violations.append(f"{prefix}:{field}") + if not is_sha256(sample.get("content_sha256")): + violations.append(f"{prefix}:content_sha256") + subgroups = sample.get("subgroups") + if not isinstance(subgroups, dict) or set(subgroups) != ( + REQUIRED_SUBGROUP_DIMENSIONS + ): + violations.append(f"{prefix}:subgroups") + else: + for dimension, value in subgroups.items(): + if dimension == "resolution": + if ( + isinstance(value, bool) + or not isinstance(value, (int, float)) + or not math.isfinite(float(value)) + or float(value) <= 0 + ): + violations.append(f"{prefix}:subgroup:resolution") + elif not isinstance(value, str) or not value.strip(): + violations.append(f"{prefix}:subgroup:{dimension}") + scopes = sample.get("authority_scopes") + if not isinstance(scopes, list) or not scopes: + violations.append(f"{prefix}:authority_scopes") + elif any( + not isinstance(scope, dict) + or set(scope) != {"task", "zone", "authority"} + or any( + not isinstance(scope.get(field), str) or not scope[field].strip() + for field in ("task", "zone", "authority") + ) + for scope in scopes + ): + violations.append(f"{prefix}:authority_scope_contract") + + if split in EVALUATED_PROTECTED_SPLIT_ROLES: + if not is_sha256(sample.get("label_sha256")): + violations.append(f"{prefix}:label_sha256") + if not is_sha256(sample.get("case_input_sha256")): + violations.append(f"{prefix}:case_input_sha256") + if sample.get("labels_access_policy") != "evaluation_only": + violations.append(f"{prefix}:labels_access_policy") + elif split == "challenge": + if sample.get("labels_sealed") is not True: + violations.append(f"{prefix}:challenge_labels_not_sealed") + if "label_sha256" in sample or "case_input_sha256" in sample: + violations.append(f"{prefix}:challenge_label_evidence_exposed") + + if set(split_counts) != PROTECTED_SPLIT_ROLES: + violations.append("protected_split:required_roles") + if payload.get("split_counts") != dict(sorted(split_counts.items())): + violations.append("protected_split:split_counts") + all_ids = set(samples_by_id) + evaluation_ids = { + sample_id + for sample_id, sample in samples_by_id.items() + if sample.get("split") in EVALUATED_PROTECTED_SPLIT_ROLES + } + if payload.get("sample_ids_sha256") != _canonical_id_hash(all_ids): + violations.append("protected_split:sample_ids_sha256") + if payload.get("evaluation_sample_ids_sha256") != _canonical_id_hash( + evaluation_ids + ): + violations.append("protected_split:evaluation_sample_ids_sha256") + + task_sample_ids = { + task: sorted( + sample_id + for sample_id in evaluation_ids + if samples_by_id[sample_id].get("task") == task + ) + for task in sorted(REQUIRED_EVALUATOR_TASK_FAMILIES) + } + observed_tasks = { + task for task, sample_ids in task_sample_ids.items() if sample_ids + } + if observed_tasks != REQUIRED_EVALUATOR_TASK_FAMILIES: + violations.append("protected_split:evaluator_task_family_coverage") + if payload.get("evaluated_task_families") != sorted( + REQUIRED_EVALUATOR_TASK_FAMILIES + ): + violations.append("protected_split:evaluated_task_families") + if payload.get("task_sample_ids") != task_sample_ids: + violations.append("protected_split:task_sample_ids") + if payload.get("task_sample_ids_canonical_json_sha256") != canonical_hash( + task_sample_ids + ): + violations.append("protected_split:task_sample_ids_sha256") + + for dimension in sorted(REQUIRED_SUBGROUP_DIMENSIONS): + values = { + str(samples_by_id[sample_id].get("subgroups", {}).get(dimension)) + for sample_id in evaluation_ids + } + if ( + len(values) + < SUBGROUP_RELEASE_POLICY["minimum_distinct_strata_per_dimension"] + ): + violations.append(f"protected_split:subgroup:{dimension}:not_stratified") + for value in values: + for task in sorted(REQUIRED_EVALUATOR_TASK_FAMILIES): + group_ids = [ + sample_id + for sample_id in evaluation_ids + if samples_by_id[sample_id].get("task") == task + and str( + samples_by_id[sample_id].get("subgroups", {}).get(dimension) + ) + == value + ] + if len(group_ids) < SUBGROUP_MIN_CASE_SUPPORT: + violations.append( + f"protected_split:subgroup:{dimension}:{value}:{task}:sample_support" + ) + independent_aois = { + samples_by_id[sample_id].get("aoi_id") for sample_id in group_ids + } + if len(independent_aois) < SUBGROUP_MIN_CASE_SUPPORT: + violations.append( + f"protected_split:subgroup:{dimension}:{value}:{task}:aoi_support" + ) + return violations, samples_by_id + + +def probe_local_cuda_runtime() -> dict[str, Any]: + """Independently observe the current local CUDA runtime; never trust a receipt.""" + + try: + import torch + except (ImportError, OSError) as exc: + return { + "status": "not_evaluable", + "reason": f"Local torch runtime is unavailable: {type(exc).__name__}", + } + try: + if torch.cuda.is_available() is not True: + return { + "status": "not_evaluable", + "reason": "torch.cuda.is_available() is false.", + } + device_index = int(torch.cuda.current_device()) + device = f"cuda:{device_index}" + + probe_value = float( + (torch.ones(4, device=device, dtype=torch.float32) * 2).sum().item() + ) + if probe_value != 8.0: + return { + "status": "not_evaluable", + "reason": "The independent CUDA kernel probe returned an invalid value.", + } + smi = subprocess.run( + [ + "nvidia-smi", + f"--id={device_index}", + "--query-gpu=uuid,name,driver_version", + "--format=csv,noheader,nounits", + ], + check=True, + capture_output=True, + text=True, + timeout=10, + ) + rows = [row.strip() for row in smi.stdout.splitlines() if row.strip()] + if len(rows) != 1: + return { + "status": "not_evaluable", + "reason": "nvidia-smi did not return exactly one device row.", + } + parts = [part.strip() for part in rows[0].split(",")] + if len(parts) != 3: + return { + "status": "not_evaluable", + "reason": "nvidia-smi device evidence is malformed.", + } + gpu_uuid, device_name, driver_version = parts + cuda_runtime = getattr(torch.version, "cuda", None) + if ( + not gpu_uuid.startswith("GPU-") + or not device_name + or not driver_version + or not isinstance(cuda_runtime, str) + or not cuda_runtime + ): + return { + "status": "not_evaluable", + "reason": "Independent CUDA identity is incomplete.", + } + return { + "status": "pass", + "device": device, + "device_name": device_name, + "gpu_uuid": gpu_uuid, + "driver_version": driver_version, + "cuda_runtime_version": cuda_runtime, + "torch_version": str(torch.__version__), + "cuda_device_count": int(torch.cuda.device_count()), + "kernel_execution_confirmed": True, + } + except (OSError, RuntimeError, subprocess.SubprocessError, ValueError) as exc: + return { + "status": "not_evaluable", + "reason": f"Independent CUDA verification failed: {type(exc).__name__}", + } + + +def _validate_inference_evidence( + payload: dict[str, Any], + manifest: dict[str, Any], + configuration: dict[str, Any], + protected_samples: dict[str, dict[str, Any]], + protected_split_hash: Any, + runtime_observation: dict[str, Any], +) -> tuple[list[str], str, dict[str, Any]]: + violations: list[str] = [] + if payload.get("schema_version") != 2: + violations.append("inference_evidence:schema_version") + if payload.get("artifact_role") != "governed_cuda_inference_execution": + violations.append("inference_evidence:artifact_role") + bindings = { + "active_model_sha256": manifest.get("active_model_sha256"), + "configuration_sha256": manifest.get("configuration_sha256"), + "evaluator_sha256": manifest.get("evaluator_sha256"), + "protected_split_sha256": protected_split_hash, + } + for field, expected in bindings.items(): + if payload.get(field) != expected: + violations.append(f"inference_evidence:{field}_mismatch") + inference = manifest.get("inference") + inference = inference if isinstance(inference, dict) else {} + if payload.get("execution_id") != inference.get("execution_id"): + violations.append("inference_evidence:execution_id_mismatch") + if payload.get("executed") is not True or payload.get("exit_code") != 0: + violations.append("inference_evidence:execution_not_successful") + if payload.get("test_used_for_selection") is not False: + violations.append("inference_evidence:test_used_for_selection") + if payload.get("device_type") != "cuda": + violations.append("inference_evidence:device_type") + device = payload.get("device") + if ( + device != inference.get("device") + or re.fullmatch(r"cuda:\d+", str(device or "")) is None + ): + violations.append("inference_evidence:device") + if payload.get("torch_cuda_is_available") is not True: + violations.append("inference_evidence:torch_cuda_unavailable") + device_count = payload.get("cuda_device_count") + if ( + not isinstance(device_count, int) + or isinstance(device_count, bool) + or device_count < 1 + ): + violations.append("inference_evidence:cuda_device_count") + if payload.get("kernel_execution_confirmed") is not True: + violations.append("inference_evidence:kernel_not_confirmed") + if payload.get("batch_failure_count") != 0: + violations.append("inference_evidence:batch_failures") + for field in ("torch_version", "cuda_runtime_version", "driver_version"): + if not isinstance(payload.get(field), str) or not payload[field].strip(): + violations.append(f"inference_evidence:{field}") + started = parse_timestamp(payload.get("started_at")) + finished = parse_timestamp(payload.get("finished_at")) + frozen = parse_timestamp(configuration.get("frozen_at")) + if started is None or finished is None or finished <= started: + violations.append("inference_evidence:timestamps") + if frozen is None or started is None or frozen >= started: + violations.append("inference_evidence:configuration_not_pre_registered") + + nvidia_smi = payload.get("nvidia_smi") + if not isinstance(nvidia_smi, dict): + violations.append("inference_evidence:nvidia_smi") + nvidia_smi = {} + else: + gpu_uuid = nvidia_smi.get("gpu_uuid") + if ( + not isinstance(gpu_uuid, str) + or not gpu_uuid.startswith("GPU-") + or len(gpu_uuid) <= 4 + ): + violations.append("inference_evidence:nvidia_smi_gpu_uuid") + for field in ("device_name", "driver_version", "cuda_version"): + if ( + not isinstance(nvidia_smi.get(field), str) + or not nvidia_smi[field].strip() + ): + violations.append(f"inference_evidence:nvidia_smi_{field}") + if not is_sha256(nvidia_smi.get("query_output_sha256")): + violations.append("inference_evidence:nvidia_smi_output_sha256") + + expected_ids = { + sample_id + for sample_id, sample in protected_samples.items() + if sample.get("split") in EVALUATED_PROTECTED_SPLIT_ROLES + } + processed_ids = payload.get("processed_sample_ids") + if ( + not isinstance(processed_ids, list) + or any(not isinstance(item, str) for item in processed_ids) + or len(processed_ids) != len(set(processed_ids)) + or set(processed_ids) != expected_ids + ): + violations.append("inference_evidence:processed_sample_ids") + if payload.get("processed_sample_ids_sha256") != _canonical_id_hash(expected_ids): + violations.append("inference_evidence:processed_sample_ids_sha256") + if payload.get("successful_sample_count") != len(expected_ids): + violations.append("inference_evidence:successful_sample_count") + + runtime_status = runtime_observation.get("status") + if runtime_status not in {"pass", "not_evaluable"}: + violations.append("inference_evidence:independent_runtime_status") + runtime_status = "fail" + if runtime_status == "pass": + expected_runtime = { + "device": payload.get("device"), + "device_name": nvidia_smi.get("device_name"), + "gpu_uuid": nvidia_smi.get("gpu_uuid"), + "driver_version": payload.get("driver_version"), + "cuda_runtime_version": payload.get("cuda_runtime_version"), + "torch_version": payload.get("torch_version"), + "cuda_device_count": payload.get("cuda_device_count"), + "kernel_execution_confirmed": True, + } + observed_runtime = { + field: runtime_observation.get(field) for field in expected_runtime + } + if observed_runtime != expected_runtime: + violations.append("inference_evidence:independent_runtime_mismatch") + return violations, str(runtime_status), runtime_observation + + +def _validate_authoritative_reference_portfolio( + payload: dict[str, Any], + protected_samples: dict[str, dict[str, Any]], + protected_split_hash: Any, + authority_requirements: list[dict[str, Any]], + evaluation: dict[str, Any] | None = None, +) -> tuple[list[str], dict[str, Any]]: + """Validate authority scope coverage and bind every evaluator reference payload.""" + + violations: list[str] = [] + if payload.get("schema_version") != 2: + violations.append("authoritative_reference:schema_version") + if payload.get("artifact_role") != "authoritative_reference_portfolio": + violations.append("authoritative_reference:artifact_role") + if payload.get("protected_split_sha256") != protected_split_hash: + violations.append("authoritative_reference:protected_split_hash_mismatch") + if ( + not isinstance(payload.get("portfolio_id"), str) + or not payload["portfolio_id"].strip() + ): + violations.append("authoritative_reference:portfolio_id") + + required = { + (item["task"], item["zone"]): item["primary"] for item in authority_requirements + } + expected_evaluation_ids = { + sample_id + for sample_id, sample in protected_samples.items() + if sample.get("split") in EVALUATED_PROTECTED_SPLIT_ROLES + } + entries = payload.get("entries") + observed: dict[tuple[str, str], str] = {} + referenced_ids: set[str] = set() + if not isinstance(entries, list) or not entries: + violations.append("authoritative_reference:entries") + entries = [] + elif payload.get("entries_canonical_json_sha256") != canonical_hash(entries): + violations.append("authoritative_reference:entries_sha256") + + for index, entry in enumerate(entries): + prefix = f"authoritative_reference:entry:{index}" + if not isinstance(entry, dict): + violations.append(f"{prefix}:not_object") + continue + key = (entry.get("task"), entry.get("zone")) + authority = entry.get("authority") + if not all(isinstance(value, str) and value.strip() for value in key): + violations.append(f"{prefix}:task_zone") + elif key in observed: + violations.append(f"{prefix}:duplicate_task_zone") + else: + observed[(str(key[0]), str(key[1]))] = str(authority) + if entry.get("source_classification") != "authoritative": + violations.append(f"{prefix}:source_classification") + if key in required and authority != required[key]: + violations.append(f"{prefix}:authority_mismatch") + if ( + not isinstance(entry.get("source_snapshot_id"), str) + or not entry["source_snapshot_id"].strip() + ): + violations.append(f"{prefix}:source_snapshot_id") + if not is_sha256(entry.get("source_snapshot_sha256")): + violations.append(f"{prefix}:source_snapshot_sha256") + sample_ids = entry.get("sample_ids") + if ( + not isinstance(sample_ids, list) + or not sample_ids + or any(not isinstance(item, str) for item in sample_ids) + or len(sample_ids) != len(set(sample_ids)) + ): + violations.append(f"{prefix}:sample_ids") + continue + scope = { + "task": entry.get("task"), + "zone": entry.get("zone"), + "authority": authority, + } + for sample_id in sample_ids: + sample = protected_samples.get(sample_id) + if ( + sample is None + or sample.get("split") not in EVALUATED_PROTECTED_SPLIT_ROLES + ): + violations.append(f"{prefix}:unknown_or_unevaluated_sample:{sample_id}") + continue + referenced_ids.add(sample_id) + if scope not in (sample.get("authority_scopes") or []): + violations.append( + f"{prefix}:sample_authority_scope_mismatch:{sample_id}" + ) + + for key, authority in required.items(): + if observed.get(key) != authority: + violations.append( + f"authoritative_reference:missing_requirement:{key[0]}:{key[1]}:{authority}" + ) + if referenced_ids != expected_evaluation_ids: + violations.append( + "authoritative_reference:sample_coverage:" + f"missing={','.join(sorted(expected_evaluation_ids - referenced_ids))}:" + f"unexpected={','.join(sorted(referenced_ids - expected_evaluation_ids))}" + ) + + sample_references = payload.get("sample_references") + sample_references_by_id: dict[str, dict[str, Any]] = {} + if not isinstance(sample_references, list): + violations.append("authoritative_reference:sample_references") + sample_references = [] + elif payload.get("sample_references_canonical_json_sha256") != canonical_hash( + sample_references + ): + violations.append("authoritative_reference:sample_references_sha256") + reference_fields = { + "sample_id", + "task", + "reference_payload_sha256", + "reference_lineage_sha256", + } + for index, item in enumerate(sample_references): + prefix = f"authoritative_reference:sample_reference:{index}" + if not isinstance(item, dict): + violations.append(f"{prefix}:not_object") + continue + if set(item) != reference_fields: + violations.append(f"{prefix}:fields") + sample_id = item.get("sample_id") + if ( + not isinstance(sample_id, str) + or sample_id not in expected_evaluation_ids + or sample_id in sample_references_by_id + ): + violations.append(f"{prefix}:sample_id") + continue + sample_references_by_id[sample_id] = item + if item.get("task") != protected_samples[sample_id].get("task"): + violations.append(f"{prefix}:task") + for field in ("reference_payload_sha256", "reference_lineage_sha256"): + if not is_sha256(item.get(field)): + violations.append(f"{prefix}:{field}") + if set(sample_references_by_id) != expected_evaluation_ids: + violations.append("authoritative_reference:sample_reference_coverage") + + if evaluation is not None: + results_by_id = { + result.get("sample_id"): result + for result in evaluation.get("results", []) + if isinstance(result, dict) and isinstance(result.get("sample_id"), str) + } + for sample_id in sorted(expected_evaluation_ids): + result = results_by_id.get(sample_id) + provided = sample_references_by_id.get(sample_id) + if result is None or provided is None: + continue + raw = result.get("raw") if isinstance(result.get("raw"), dict) else {} + hashes = raw.get("hashes") if isinstance(raw.get("hashes"), dict) else {} + lineage = ( + raw.get("input_lineage") + if isinstance(raw.get("input_lineage"), dict) + else {} + ) + expected_reference = { + "sample_id": sample_id, + "task": result.get("task"), + "reference_payload_sha256": hashes.get( + "references_canonical_json_sha256" + ), + "reference_lineage_sha256": canonical_hash(lineage.get("reference")), + } + if provided != expected_reference: + violations.append( + f"authoritative_reference:sample_reference_mismatch:{sample_id}" + ) + + evidence = { + "required": [ + {"task": task, "zone": zone, "authority": authority} + for (task, zone), authority in sorted(required.items()) + ], + "observed": [ + {"task": task, "zone": zone, "authority": authority} + for (task, zone), authority in sorted(observed.items()) + if (task, zone) in required + ], + "covered_evaluation_sample_count": len(referenced_ids), + "reference_bound_sample_count": len(sample_references_by_id), + "required_evaluation_sample_count": len(expected_evaluation_ids), + } + return violations, evidence + + +def _evaluate_raw_product_portfolio( + repo_root: Path, + portfolio_path: Path, + payload: dict[str, Any], + manifest: dict[str, Any], + configuration: dict[str, Any], + protected_samples: dict[str, dict[str, Any]], + protected_split_hash: Any, + authority_reference_hash: Any, + inference_evidence_hash: Any, +) -> tuple[list[str], dict[str, Any] | None]: + """Cross-bind product cases and recompute metrics with the trusted evaluator.""" + + violations: list[str] = [] + required_fields = { + "schema_version", + "portfolio_kind", + "portfolio_id", + "claim_boundary", + "split_roles", + "selection_policy", + "protected_policy", + "portfolio_lineage", + "cases", + } + if set(payload) != required_fields: + violations.append("raw_predictions:portfolio_fields") + if payload.get("schema_version") != 2: + violations.append("raw_predictions:schema_version") + if payload.get("portfolio_kind") != "governed_product_baseline": + violations.append("raw_predictions:portfolio_kind") + if payload.get("portfolio_id") != manifest.get("baseline_id"): + violations.append("raw_predictions:portfolio_id") + claim = payload.get("claim_boundary") + claim_lower = claim.lower() if isinstance(claim, str) else "" + if "governed product baseline" not in claim_lower or "synthetic" in claim_lower: + violations.append("raw_predictions:claim_boundary") + if ( + payload.get("selection_policy") + != "frozen_validation_calibration_only_no_protected_selection" + ): + violations.append("raw_predictions:selection_policy") + if payload.get("split_roles") != ["test", "background-test"]: + violations.append("raw_predictions:split_roles") + if payload.get("protected_policy") != EXPECTED_PROTECTED_POLICY: + violations.append("raw_predictions:protected_policy") + try: + source_path = ( + portfolio_path.resolve().relative_to(repo_root.resolve()).as_posix() + ) + except (OSError, ValueError): + source_path = str(portfolio_path) + expected_lineage = { + "origin": "governed_product_inference", + "source_path": source_path, + "version": manifest.get("baseline_id"), + "active_model_sha256": manifest.get("active_model_sha256"), + "configuration_sha256": manifest.get("configuration_sha256"), + "protected_split_sha256": protected_split_hash, + "authoritative_reference_sha256": authority_reference_hash, + "inference_evidence_sha256": inference_evidence_hash, + } + if payload.get("portfolio_lineage") != expected_lineage: + violations.append("raw_predictions:portfolio_lineage") + + expected_ids = { + sample_id + for sample_id, sample in protected_samples.items() + if sample.get("split") in EVALUATED_PROTECTED_SPLIT_ROLES + } + cases = payload.get("cases") + cases_by_id: dict[str, dict[str, Any]] = {} + if not isinstance(cases, list) or not cases: + violations.append("raw_predictions:cases") + cases = [] + for index, case in enumerate(cases): + prefix = f"raw_predictions:case:{index}" + if not isinstance(case, dict): + violations.append(f"{prefix}:not_object") + continue + sample_id = case.get("sample_id") + if ( + not isinstance(sample_id, str) + or sample_id not in expected_ids + or sample_id in cases_by_id + ): + violations.append(f"{prefix}:sample_id") + continue + cases_by_id[sample_id] = case + sample = protected_samples[sample_id] + for field in ("task", "split"): + if case.get(field) != sample.get(field): + violations.append(f"{prefix}:{field}_mismatch") + metadata = case.get("metadata") + if not isinstance(metadata, dict): + violations.append(f"{prefix}:metadata") + else: + for dimension, field in REQUIRED_SUBGROUP_DIMENSION_FIELDS.items(): + if metadata.get(field) != sample.get("subgroups", {}).get(dimension): + violations.append(f"{prefix}:metadata:{field}_mismatch") + parameters = configuration.get("parameters_by_task") + expected_config = ( + parameters.get(sample.get("task")) if isinstance(parameters, dict) else None + ) + if case.get("config") != expected_config: + violations.append(f"{prefix}:configuration_mismatch") + if canonical_hash(case) != sample.get("case_input_sha256"): + violations.append(f"{prefix}:case_input_sha256_mismatch") + if set(cases_by_id) != expected_ids: + violations.append( + "raw_predictions:sample_coverage:" + f"missing={','.join(sorted(expected_ids - set(cases_by_id)))}:" + f"unexpected={','.join(sorted(set(cases_by_id) - expected_ids))}" + ) + if { + case.get("task") for case in cases_by_id.values() + } != REQUIRED_EVALUATOR_TASK_FAMILIES: + violations.append("raw_predictions:evaluator_task_family_coverage") + + evaluation: dict[str, Any] | None = None + try: + evaluation = evaluate_cases(portfolio_path, expected_ids) + except (OSError, ValueError, TypeError, KeyError) as exc: + violations.append( + f"raw_predictions:evaluator_rejected:{type(exc).__name__}:{str(exc)}" + ) + if evaluation is not None: + if evaluation.get("portfolio_kind") != "governed_product_baseline": + violations.append("raw_predictions:evaluation_portfolio_kind") + if ( + set(evaluation.get("evaluated_task_families") or []) + != REQUIRED_EVALUATOR_TASK_FAMILIES + ): + violations.append("raw_predictions:evaluation_task_family_coverage") + if evaluation.get("task_count") != len(REQUIRED_EVALUATOR_TASK_FAMILIES): + violations.append("raw_predictions:evaluation_task_count") + if evaluation.get("case_count") != len(expected_ids): + violations.append("raw_predictions:evaluation_case_count") + if evaluation.get("task_inventory") != task_inventory(): + violations.append("raw_predictions:evaluation_task_inventory") + for result in evaluation.get("results", []): + if not isinstance(result, dict): + violations.append("raw_predictions:evaluation_result_not_object") + continue + sample_id = result.get("sample_id") + raw = result.get("raw") if isinstance(result.get("raw"), dict) else {} + sample = protected_samples.get(str(sample_id), {}) + if raw.get("input_sha256") != sample.get("case_input_sha256"): + violations.append( + f"raw_predictions:evaluation_input_sha256_mismatch:{sample_id}" + ) + return violations, evaluation + + +def _append_metric_numeric_sanity_violations( + value: Any, + violations: list[str], + path: str = "metric_report", +) -> None: + """Reject impossible reported numbers before exact recomputation comparison.""" + + if isinstance(value, dict): + for key, child in value.items(): + child_path = f"{path}.{key}" + if isinstance(child, bool): + continue + if isinstance(child, (int, float)): + numeric = float(child) + if not math.isfinite(numeric): + violations.append(f"metric_report:non_finite:{child_path}") + continue + normalized = key.lower() + count_like = ( + normalized in {"support", "case_support", "minimum_case_support"} + or normalized.endswith("_count") + or normalized.endswith("_pixels") + or normalized + in {"true_positive", "false_positive", "false_negative"} + ) + if count_like and ( + numeric < 0.0 + or (".macro." not in child_path and type(child) is not int) + ): + violations.append( + f"metric_report:invalid_support_or_count:{child_path}" + ) + rate_like = normalized in { + "precision", + "recall", + "f1", + "iou", + "mean_iou", + "dice", + "mean_dice", + "boundary_f1", + "mean_boundary_f1", + "accuracy", + "prediction_coverage", + "reference_coverage", + "coverage", + "retained_prediction_coverage", + "risk", + "false_discovery_rate", + "miss_rate", + "ap50", + "ap50_95", + "map50", + "map50_95", + "ece", + "brier", + } + if rate_like and not 0.0 <= numeric <= 1.0: + violations.append(f"metric_report:impossible_rate:{child_path}") + _append_metric_numeric_sanity_violations(child, violations, child_path) + lower = value.get("lower") + upper = value.get("upper") + support = value.get("support") + if isinstance(support, bool) or ( + support is not None and (not isinstance(support, int) or support < 0) + ): + violations.append(f"metric_report:invalid_ci_support:{path}") + if ( + isinstance(lower, (int, float)) + and not isinstance(lower, bool) + and isinstance(upper, (int, float)) + and not isinstance(upper, bool) + ): + if float(lower) > float(upper): + violations.append(f"metric_report:invalid_ci_order:{path}") + if "wilson" in path.lower() and (float(lower) < 0.0 or float(upper) > 1.0): + violations.append(f"metric_report:impossible_wilson_ci:{path}") + elif isinstance(value, list): + for index, child in enumerate(value): + _append_metric_numeric_sanity_violations( + child, violations, f"{path}[{index}]" + ) + elif isinstance(value, float) and not math.isfinite(value): + violations.append(f"metric_report:non_finite:{path}") + + +def _validate_metric_report( + payload: dict[str, Any], + manifest: dict[str, Any], + protected_samples: dict[str, dict[str, Any]], + protected_split_hash: Any, + raw_predictions_hash: Any, + evaluation: dict[str, Any] | None, + configuration: dict[str, Any], +) -> tuple[list[str], dict[str, Any]]: + """Require exact equality with an in-process evaluator recomputation.""" + + violations: list[str] = [] + _append_metric_numeric_sanity_violations(payload, violations) + required_fields = { + "schema_version", + "artifact_role", + "active_model_sha256", + "configuration_sha256", + "evaluator_sha256", + "protected_split_sha256", + "raw_predictions_sha256", + "evaluator_version", + "portfolio_kind", + "portfolio_id", + "portfolio_file_sha256", + "portfolio_canonical_json_sha256", + "evaluated_task_families", + "task_count", + "case_count", + "task_inventory", + "task_inventory_sha256", + "results", + "results_canonical_json_sha256", + "portfolio_metrics", + "portfolio_metrics_canonical_json_sha256", + "subgroups", + "subgroups_canonical_json_sha256", + "failures", + "failures_canonical_json_sha256", + "failure_taxonomy", + "failure_taxonomy_canonical_json_sha256", + "subgroup_dimension_mapping", + "subgroup_release_policy", + "pre_registered_targets", + "pre_registered_targets_sha256", + } + if set(payload) != required_fields: + violations.append("metric_report:fields") + if payload.get("schema_version") != 3: + violations.append("metric_report:schema_version") + if payload.get("artifact_role") != "protected_metric_report": + violations.append("metric_report:artifact_role") + bindings = { + "active_model_sha256": manifest.get("active_model_sha256"), + "configuration_sha256": manifest.get("configuration_sha256"), + "evaluator_sha256": manifest.get("evaluator_sha256"), + "protected_split_sha256": protected_split_hash, + "raw_predictions_sha256": raw_predictions_hash, + } + for field, expected in bindings.items(): + if payload.get(field) != expected: + violations.append(f"metric_report:{field}_mismatch") + if evaluation is None: + violations.append("metric_report:evaluator_result_unavailable") + return violations, { + "required_dimensions": sorted(REQUIRED_SUBGROUP_DIMENSIONS), + "policy": SUBGROUP_RELEASE_POLICY, + "status": "not_evaluable", + } + + exact_fields = ( + "evaluator_version", + "portfolio_kind", + "portfolio_id", + "portfolio_file_sha256", + "portfolio_canonical_json_sha256", + "evaluated_task_families", + "task_count", + "case_count", + "task_inventory", + "results", + "results_canonical_json_sha256", + "portfolio_metrics", + "subgroups", + "failures", + "failure_taxonomy", + ) + for field in exact_fields: + if payload.get(field) != evaluation.get(field): + violations.append(f"metric_report:{field}_recomputation_mismatch") + for hash_field, value_field in { + "task_inventory_sha256": "task_inventory", + "portfolio_metrics_canonical_json_sha256": "portfolio_metrics", + "subgroups_canonical_json_sha256": "subgroups", + "failures_canonical_json_sha256": "failures", + "failure_taxonomy_canonical_json_sha256": "failure_taxonomy", + }.items(): + if payload.get(hash_field) != canonical_hash(evaluation.get(value_field)): + violations.append(f"metric_report:{hash_field}") + results = payload.get("results") + if not isinstance(results, list) or payload.get( + "results_canonical_json_sha256" + ) != canonical_hash(results): + violations.append("metric_report:results_sha256") + if ( + set(payload.get("evaluated_task_families") or []) + != REQUIRED_EVALUATOR_TASK_FAMILIES + ): + violations.append("metric_report:evaluator_task_family_coverage") + expected_ids = { + sample_id + for sample_id, sample in protected_samples.items() + if sample.get("split") in EVALUATED_PROTECTED_SPLIT_ROLES + } + result_ids = { + item.get("sample_id") + for item in payload.get("results", []) + if isinstance(item, dict) + } + if result_ids != expected_ids: + violations.append("metric_report:sample_coverage") + + reported_portfolio_metrics = payload.get("portfolio_metrics") + if isinstance(reported_portfolio_metrics, dict): + for task, metrics in reported_portfolio_metrics.items(): + support = ( + metrics.get("observation_support") + if isinstance(metrics, dict) + else None + ) + support_values = ( + [value for value in support.values() if type(value) is int] + if isinstance(support, dict) + else [] + ) + if not support_values or sum(support_values) <= 0: + violations.append(f"metric_report:task:{task}:empty_support") + + if payload.get("subgroup_dimension_mapping") != REQUIRED_SUBGROUP_DIMENSION_FIELDS: + violations.append("metric_report:subgroup_dimension_mapping") + if payload.get("subgroup_release_policy") != SUBGROUP_RELEASE_POLICY: + violations.append("metric_report:subgroup_release_policy") + subgroups = evaluation.get("subgroups") or {} + dimension_reports = subgroups.get("dimensions") + if not isinstance(dimension_reports, dict) or set(dimension_reports) != set( + REQUIRED_SUBGROUP_DIMENSION_FIELDS.values() + ): + violations.append("metric_report:subgroup_dimension_coverage") + dimension_reports = {} + for dimension, evaluator_field in REQUIRED_SUBGROUP_DIMENSION_FIELDS.items(): + report = dimension_reports.get(evaluator_field) + if not isinstance(report, dict): + violations.append(f"metric_report:subgroup:{dimension}:missing") + continue + strata = report.get("strata") + if ( + not isinstance(strata, dict) + or len(strata) + < SUBGROUP_RELEASE_POLICY["minimum_distinct_strata_per_dimension"] + ): + violations.append(f"metric_report:subgroup:{dimension}:not_stratified") + continue + for stratum, evidence in strata.items(): + task_metrics = ( + evidence.get("task_metrics") if isinstance(evidence, dict) else None + ) + if ( + not isinstance(task_metrics, dict) + or set(task_metrics) != REQUIRED_EVALUATOR_TASK_FAMILIES + ): + violations.append( + f"metric_report:subgroup:{dimension}:{stratum}:task_coverage" + ) + continue + for task, task_metric in task_metrics.items(): + if ( + not isinstance(task_metric, dict) + or task_metric.get("status") != "evaluable" + or not isinstance(task_metric.get("case_support"), int) + or task_metric["case_support"] < SUBGROUP_MIN_CASE_SUPPORT + or (task_metric.get("primary_metric") or {}).get("value") is None + ): + violations.append( + f"metric_report:subgroup:{dimension}:{stratum}:{task}:support" + ) + worst = report.get("worst_stratum_by_task") + if ( + not isinstance(worst, dict) + or set(worst) != REQUIRED_EVALUATOR_TASK_FAMILIES + ): + violations.append(f"metric_report:subgroup:{dimension}:worst_strata") + elif any( + not isinstance(item, dict) or item.get("status") != "computed" + for item in worst.values() + ): + violations.append(f"metric_report:subgroup:{dimension}:worst_not_computed") + + portfolio_metrics = evaluation.get("portfolio_metrics") + if ( + not isinstance(portfolio_metrics, dict) + or set(portfolio_metrics) != REQUIRED_EVALUATOR_TASK_FAMILIES + ): + violations.append("metric_report:portfolio_metric_task_coverage") + portfolio_metrics = {} + for task, metrics in portfolio_metrics.items(): + support = ( + metrics.get("observation_support") if isinstance(metrics, dict) else {} + ) + support_values = ( + [value for value in support.values() if type(value) is int] + if isinstance(support, dict) + else [] + ) + if not support_values or sum(support_values) <= 0: + violations.append(f"metric_report:task:{task}:empty_support") + if ( + type(metrics.get("case_support")) is not int + or metrics["case_support"] < SUBGROUP_MIN_CASE_SUPPORT * 2 + ): + violations.append(f"metric_report:task:{task}:case_support") + + targets = configuration.get("subgroup_release_targets") + if payload.get("pre_registered_targets") != targets: + violations.append("metric_report:pre_registered_targets") + if payload.get("pre_registered_targets_sha256") != canonical_hash(targets): + violations.append("metric_report:pre_registered_targets_sha256") + if ( + not isinstance(targets, dict) + or set(targets) != REQUIRED_EVALUATOR_TASK_FAMILIES + ): + violations.append("metric_report:release_target_coverage") + else: + for task, target in targets.items(): + primary = (portfolio_metrics.get(task) or {}).get("primary_metric") or {} + if not isinstance(target, dict) or set(target) != { + "metric", + "direction", + "threshold", + }: + violations.append(f"metric_report:release_target:{task}:contract") + continue + if target.get("metric") != primary.get("name"): + violations.append(f"metric_report:release_target:{task}:metric") + if target.get("direction") != primary.get("direction"): + violations.append(f"metric_report:release_target:{task}:direction") + threshold = target.get("threshold") + current = primary.get("value") + if ( + isinstance(threshold, bool) + or not isinstance(threshold, (int, float)) + or not math.isfinite(float(threshold)) + or float(threshold) < 0.0 + ): + violations.append(f"metric_report:release_target:{task}:threshold") + continue + if ( + isinstance(current, bool) + or not isinstance(current, (int, float)) + or not math.isfinite(float(current)) + ): + violations.append(f"metric_report:release_target:{task}:not_evaluable") + continue + direction = target.get("direction") + if ( + direction == "higher_is_better" and float(current) < float(threshold) + ) or (direction == "lower_is_better" and float(current) > float(threshold)): + violations.append(f"metric_report:release_target_not_met:{task}") + + return violations, { + "required_dimensions": sorted(REQUIRED_SUBGROUP_DIMENSIONS), + "dimension_mapping": REQUIRED_SUBGROUP_DIMENSION_FIELDS, + "policy": SUBGROUP_RELEASE_POLICY, + "task_families": sorted(REQUIRED_EVALUATOR_TASK_FAMILIES), + "evaluator_version": EVALUATOR_VERSION, + "subgroups_sha256": canonical_hash(subgroups), + "portfolio_metrics_sha256": canonical_hash(portfolio_metrics), + } + + +def _validate_human_review_ledger( + payload: dict[str, Any], + protected_samples: dict[str, dict[str, Any]], + protected_split_hash: Any, + raw_predictions_hash: Any, +) -> tuple[list[str], dict[str, Any]]: + violations: list[str] = [] + if payload.get("schema_version") != 1: + violations.append("human_review:schema_version") + if payload.get("artifact_role") != "human_review_ledger": + violations.append("human_review:artifact_role") + if payload.get("protected_split_sha256") != protected_split_hash: + violations.append("human_review:protected_split_hash_mismatch") + if payload.get("raw_predictions_sha256") != raw_predictions_hash: + violations.append("human_review:raw_predictions_hash_mismatch") + expected_ids = { + sample_id + for sample_id, sample in protected_samples.items() + if sample.get("split") in EVALUATED_PROTECTED_SPLIT_ROLES + } + entries = payload.get("entries") + if not isinstance(entries, list): + violations.append("human_review:entries") + entries = [] + elif payload.get("entries_canonical_json_sha256") != canonical_hash(entries): + violations.append("human_review:entries_sha256") + by_id: dict[str, dict[str, Any]] = {} + fields = { + "sample_id", + "reviewer_id", + "review_timestamp", + "decision", + "label_sha256", + "case_input_sha256", + "entry_canonical_json_sha256", + } + for index, entry in enumerate(entries): + prefix = f"human_review:entry:{index}" + if not isinstance(entry, dict) or set(entry) != fields: + violations.append(f"{prefix}:contract") + continue + sample_id = entry.get("sample_id") + if ( + not isinstance(sample_id, str) + or sample_id not in expected_ids + or sample_id in by_id + ): + violations.append(f"{prefix}:sample_id") + continue + by_id[sample_id] = entry + sample = protected_samples[sample_id] + if ( + not isinstance(entry.get("reviewer_id"), str) + or not entry["reviewer_id"].strip() + or entry.get("decision") != "accepted" + or parse_timestamp(entry.get("review_timestamp")) is None + ): + violations.append(f"{prefix}:human_acceptance") + if entry.get("label_sha256") != sample.get("label_sha256"): + violations.append(f"{prefix}:label_sha256") + if entry.get("case_input_sha256") != sample.get("case_input_sha256"): + violations.append(f"{prefix}:case_input_sha256") + hash_input = { + key: value + for key, value in entry.items() + if key != "entry_canonical_json_sha256" + } + if entry.get("entry_canonical_json_sha256") != canonical_hash(hash_input): + violations.append(f"{prefix}:entry_sha256") + if set(by_id) != expected_ids: + violations.append("human_review:sample_coverage") + return violations, { + "reviewed_sample_count": len(by_id), + "required_sample_count": len(expected_ids), + "ledger_sha256": canonical_hash(entries), + } + + +def _validate_geometric_leakage_audit( + payload: dict[str, Any], + manifest: dict[str, Any], + protected_samples: dict[str, dict[str, Any]], + protected_split_hash: Any, + raw_predictions_hash: Any, +) -> tuple[list[str], dict[str, Any]]: + violations: list[str] = [] + if payload.get("schema_version") != 1: + violations.append("geometric_leakage:schema_version") + if payload.get("artifact_role") != "geometric_leakage_audit": + violations.append("geometric_leakage:artifact_role") + for field, expected in { + "protected_split_sha256": protected_split_hash, + "raw_predictions_sha256": raw_predictions_hash, + "development_split_manifest_sha256": manifest.get( + "development_split_manifest_sha256" + ), + }.items(): + if payload.get(field) != expected: + violations.append(f"geometric_leakage:{field}_mismatch") + expected_ids = { + sample_id + for sample_id, sample in protected_samples.items() + if sample.get("split") in EVALUATED_PROTECTED_SPLIT_ROLES + } + if payload.get("evaluation_sample_ids_sha256") != _canonical_id_hash(expected_ids): + violations.append("geometric_leakage:evaluation_sample_ids_sha256") + if payload.get("distance_threshold_m") != 2000: + violations.append("geometric_leakage:distance_threshold_m") + crs = payload.get("projected_crs") + if not isinstance(crs, str) or not crs.strip() or "4326" in crs: + violations.append("geometric_leakage:projected_crs") + algorithm = payload.get("algorithm") + if not isinstance(algorithm, str) or "geometry" not in algorithm.lower(): + violations.append("geometric_leakage:algorithm") + if ( + payload.get("below_threshold_pairs") != [] + or payload.get("below_threshold_pair_count") != 0 + ): + violations.append("geometric_leakage:pairs_below_2000_m") + minimum_distance = payload.get("minimum_observed_distance_m") + if ( + isinstance(minimum_distance, bool) + or not isinstance(minimum_distance, (int, float)) + or not math.isfinite(float(minimum_distance)) + or float(minimum_distance) < 2000.0 + ): + violations.append("geometric_leakage:minimum_observed_distance_m") + return violations, { + "distance_threshold_m": 2000, + "below_threshold_pair_count": payload.get("below_threshold_pair_count"), + "minimum_observed_distance_m": minimum_distance, + "evaluation_sample_ids_sha256": _canonical_id_hash(expected_ids), + } + + +def _validate_vault_access_evidence( + payload: dict[str, Any], + manifest: dict[str, Any], + protected_samples: dict[str, dict[str, Any]], + protected_split_hash: Any, + raw_predictions_hash: Any, +) -> tuple[list[str], dict[str, Any]]: + violations: list[str] = [] + if payload.get("schema_version") != 1: + violations.append("vault_access:schema_version") + if payload.get("artifact_role") != "vault_access_evidence": + violations.append("vault_access:artifact_role") + if payload.get("protected_split_sha256") != protected_split_hash: + violations.append("vault_access:protected_split_hash_mismatch") + if payload.get("raw_predictions_sha256") != raw_predictions_hash: + violations.append("vault_access:raw_predictions_hash_mismatch") + inference = ( + manifest.get("inference") if isinstance(manifest.get("inference"), dict) else {} + ) + if payload.get("execution_id") != inference.get("execution_id"): + violations.append("vault_access:execution_id_mismatch") + if payload.get("vault_mode") != "read_only_evaluation": + violations.append("vault_access:vault_mode") + if payload.get("challenge_labels_accessed") is not False: + violations.append("vault_access:challenge_labels_accessed") + expected_ids = { + sample_id + for sample_id, sample in protected_samples.items() + if sample.get("split") in EVALUATED_PROTECTED_SPLIT_ROLES + } + challenge_ids = { + sample_id + for sample_id, sample in protected_samples.items() + if sample.get("split") == "challenge" + } + entries = payload.get("access_log") + if not isinstance(entries, list) or not entries: + violations.append("vault_access:access_log") + entries = [] + elif payload.get("access_log_canonical_json_sha256") != canonical_hash(entries): + violations.append("vault_access:access_log_sha256") + previous = "0" * 64 + accessed: set[str] = set() + fields = { + "sequence", + "timestamp", + "actor", + "purpose", + "operation", + "sample_ids", + "previous_entry_sha256", + "entry_sha256", + } + for index, entry in enumerate(entries, start=1): + prefix = f"vault_access:entry:{index}" + if not isinstance(entry, dict) or set(entry) != fields: + violations.append(f"{prefix}:contract") + continue + if entry.get("sequence") != index: + violations.append(f"{prefix}:sequence") + if parse_timestamp(entry.get("timestamp")) is None: + violations.append(f"{prefix}:timestamp") + if entry.get("actor") != "phase4-evaluator": + violations.append(f"{prefix}:actor") + if entry.get("purpose") != "evaluation_only": + violations.append(f"{prefix}:purpose") + if entry.get("operation") != "read": + violations.append(f"{prefix}:operation") + sample_ids = entry.get("sample_ids") + if ( + not isinstance(sample_ids, list) + or any(not isinstance(item, str) for item in sample_ids) + or len(sample_ids) != len(set(sample_ids)) + ): + violations.append(f"{prefix}:sample_ids") + sample_ids = [] + if set(sample_ids) & challenge_ids: + violations.append(f"{prefix}:challenge_access") + if set(sample_ids) - expected_ids: + violations.append(f"{prefix}:unexpected_sample") + accessed.update(sample_ids) + if entry.get("previous_entry_sha256") != previous: + violations.append(f"{prefix}:previous_entry_sha256") + hash_input = { + key: value for key, value in entry.items() if key != "entry_sha256" + } + observed_hash = canonical_hash(hash_input) + if entry.get("entry_sha256") != observed_hash: + violations.append(f"{prefix}:entry_sha256") + previous = observed_hash + if accessed != expected_ids: + violations.append("vault_access:sample_coverage") + return violations, { + "vault_mode": payload.get("vault_mode"), + "accessed_sample_count": len(accessed), + "required_sample_count": len(expected_ids), + "access_log_sha256": canonical_hash(entries), + "final_chain_sha256": previous, + } + + +def _unavailable_product_baseline_gate( + status: str, + reason: str, + **details: Any, +) -> dict[str, Any]: + if status not in {"fail", "not_evaluable"}: + raise ValueError("Unavailable product gates can only fail or be not_evaluable") + derived = { + name: {"status": status, "reason": reason} + for name in ( + "authoritative_reference_portfolio_available", + "representative_product_subgroup_support", + "human_review_complete", + "split_independence", + "protected_storage_isolation", + ) + } + return { + "status": status, + "reason": reason, + "checked_artifacts": [], + "derived_gates": derived, + **details, + } + + def product_baseline_manifest_gate( repo_root: Path, manifest_path: Path, active_model: dict[str, Any], + authority_requirements: list[dict[str, Any]] | None = None, + runtime_probe: Callable[[], dict[str, Any]] | None = None, ) -> dict[str, Any]: - relative = manifest_path + governed_root = (repo_root / "artifacts/evidence/accuracy/P4").resolve() + resolved_manifest = manifest_path.resolve() try: - relative = manifest_path.resolve().relative_to(repo_root.resolve()) + relative = resolved_manifest.relative_to(repo_root.resolve()) + resolved_manifest.relative_to(governed_root) except (OSError, ValueError): - return { - "status": "fail", - "reason": "Product baseline manifest must reside inside the governed repository evidence root.", - "path": str(manifest_path), - } - if not manifest_path.is_file(): - return { - "status": "not_evaluable", - "reason": "No executed, hash-bound product incumbent baseline manifest is available.", - "expected_path": relative.as_posix(), - } - try: - manifest = json.loads(manifest_path.read_text(encoding="utf-8")) - except (OSError, json.JSONDecodeError) as exc: - return { - "status": "fail", - "reason": f"Unreadable product baseline manifest: {exc}", - } - required_keys = { - "schema_version", - "status", - "synthetic", - "active_model_sha256", - "evaluator_sha256", - "configuration_sha256", - "protected_split_manifest", - "authoritative_reference_manifest", - "raw_predictions", - "metric_report", - "inference", - } - missing = sorted(required_keys - set(manifest)) - violations: list[str] = [] - if missing: - violations.append(f"missing_fields:{','.join(missing)}") - if manifest.get("status") != "pass": - violations.append("manifest_status_not_pass") - if manifest.get("synthetic") is not False: - violations.append("synthetic_or_unspecified") - if manifest.get("active_model_sha256") != active_model.get("sha256"): - violations.append("active_model_hash_mismatch") - evaluator_path = repo_root / "scripts/accuracy_phase4_evaluator.py" - if manifest.get("evaluator_sha256") != sha256(evaluator_path): - violations.append("evaluator_hash_mismatch") - inference = manifest.get("inference") or {} - if inference.get("executed") is not True: - violations.append("inference_not_executed") - if inference.get("test_used_for_selection") is not False: - violations.append("protected_test_selection_policy_invalid") - if not str(inference.get("device") or "").lower().startswith("cuda"): - violations.append("governed_cuda_execution_not_proven") - checked_artifacts: list[dict[str, Any]] = [] - for key in ( - "protected_split_manifest", - "authoritative_reference_manifest", - "raw_predictions", - "metric_report", - ): - item = manifest.get(key) or {} - item_path = repo_root / str(item.get("path") or "") - try: - item_path.resolve().relative_to(repo_root.resolve()) - except (OSError, ValueError): - violations.append(f"{key}_outside_repository") - continue - if not item_path.is_file(): - violations.append(f"{key}_missing") - continue - observed_hash = sha256(item_path) - checked_artifacts.append( - { - "role": key, - "path": item_path.relative_to(repo_root).as_posix(), - "sha256": observed_hash, - } + return _unavailable_product_baseline_gate( + "fail", + "Product baseline manifest must reside inside artifacts/evidence/accuracy/P4.", + path=str(manifest_path), ) - if observed_hash != item.get("sha256"): - violations.append(f"{key}_hash_mismatch") + if not resolved_manifest.is_file(): + return _unavailable_product_baseline_gate( + "not_evaluable", + "No executed, hash-bound product incumbent baseline manifest is available.", + expected_path=relative.as_posix(), + ) + try: + manifest = json.loads(resolved_manifest.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + return _unavailable_product_baseline_gate( + "fail", + f"Unreadable product baseline manifest: {exc}", + path=relative.as_posix(), + ) + if not isinstance(manifest, dict): + return _unavailable_product_baseline_gate( + "fail", + "Product baseline manifest root must be an object.", + path=relative.as_posix(), + ) + + evaluator_path = repo_root / "scripts/accuracy_phase4_evaluator.py" + if not evaluator_path.is_file(): + return _unavailable_product_baseline_gate( + "fail", + "The hash-bound Phase-4 evaluator is unavailable.", + path=relative.as_posix(), + ) + evaluator_hash = sha256(evaluator_path) + model_gate = active_model_availability_gate(active_model) + manifest_violations = _validate_product_manifest_contract( + manifest, active_model, evaluator_hash + ) + artifact_violations: list[str] = [] + payloads, checked_artifacts = _load_governed_artifacts( + repo_root, governed_root, manifest, artifact_violations + ) + + def descriptor(role: str) -> dict[str, Any]: + value = manifest.get(role) + return value if isinstance(value, dict) else {} + + if manifest.get("configuration_sha256") != descriptor("configuration").get( + "sha256" + ): + artifact_violations.append("configuration:manifest_hash_binding") + configuration = payloads.get("configuration", {}) + configuration_violations = ( + _validate_configuration(configuration, manifest) if configuration else [] + ) + protected_violations: list[str] = [] + protected_samples: dict[str, dict[str, Any]] = {} + if "protected_split_manifest" in payloads: + protected_violations, protected_samples = _validate_protected_split( + payloads["protected_split_manifest"] + ) + protected_hash = descriptor("protected_split_manifest").get("sha256") + raw_hash = descriptor("raw_predictions").get("sha256") + + probe = runtime_probe or probe_local_cuda_runtime + try: + runtime_observation = probe() + except ( + Exception + ) as exc: # pragma: no cover - defensive boundary around hardware probe + runtime_observation = { + "status": "not_evaluable", + "reason": f"Independent CUDA probe raised {type(exc).__name__}.", + } + if not isinstance(runtime_observation, dict): + runtime_observation = { + "status": "not_evaluable", + "reason": "Independent CUDA probe returned no structured evidence.", + } + inference_violations: list[str] = [] + runtime_status = str(runtime_observation.get("status")) + if "inference_evidence" in payloads: + inference_violations, runtime_status, runtime_observation = ( + _validate_inference_evidence( + payloads["inference_evidence"], + manifest, + configuration, + protected_samples, + protected_hash, + runtime_observation, + ) + ) + + raw_violations: list[str] = [] + evaluation: dict[str, Any] | None = None + if "raw_predictions" in payloads: + raw_descriptor = descriptor("raw_predictions") + raw_path_value = raw_descriptor.get("path") + raw_path = repo_root / str(raw_path_value or "") + raw_violations, evaluation = _evaluate_raw_product_portfolio( + repo_root, + raw_path, + payloads["raw_predictions"], + manifest, + configuration, + protected_samples, + protected_hash, + descriptor("authoritative_reference_manifest").get("sha256"), + descriptor("inference_evidence").get("sha256"), + ) + + requirements = authority_requirements or [ + dict(item) for item in REQUIRED_AUTHORITY_REQUIREMENTS + ] + authority_violations: list[str] = [] + authority_evidence: dict[str, Any] = {"required": requirements, "observed": []} + if "authoritative_reference_manifest" in payloads: + authority_violations, authority_evidence = ( + _validate_authoritative_reference_portfolio( + payloads["authoritative_reference_manifest"], + protected_samples, + protected_hash, + requirements, + evaluation, + ) + ) + + metric_violations: list[str] = [] + subgroup_evidence: dict[str, Any] = { + "required_dimensions": sorted(REQUIRED_SUBGROUP_DIMENSIONS), + "status": "not_evaluable", + } + if "metric_report" in payloads: + metric_violations, subgroup_evidence = _validate_metric_report( + payloads["metric_report"], + manifest, + protected_samples, + protected_hash, + raw_hash, + evaluation, + configuration, + ) + + review_violations: list[str] = [] + review_evidence: dict[str, Any] = {} + if "human_review_ledger" in payloads: + review_violations, review_evidence = _validate_human_review_ledger( + payloads["human_review_ledger"], protected_samples, protected_hash, raw_hash + ) + leakage_violations: list[str] = [] + leakage_evidence: dict[str, Any] = {} + if "geometric_leakage_audit" in payloads: + leakage_violations, leakage_evidence = _validate_geometric_leakage_audit( + payloads["geometric_leakage_audit"], + manifest, + protected_samples, + protected_hash, + raw_hash, + ) + vault_violations: list[str] = [] + vault_evidence: dict[str, Any] = {} + if "vault_access_evidence" in payloads: + vault_violations, vault_evidence = _validate_vault_access_evidence( + payloads["vault_access_evidence"], + manifest, + protected_samples, + protected_hash, + raw_hash, + ) + + model_violations = ( + [f"active_model:{model_gate.get('reason')}"] + if model_gate.get("status") == "fail" + else [] + ) + categories = { + "active_model_integrity": model_violations, + "manifest_contract": manifest_violations, + "artifact_integrity": artifact_violations, + "configuration": configuration_violations, + "protected_split": protected_violations, + "cuda_inference": inference_violations, + "authoritative_reference": authority_violations, + "raw_predictions_and_recomputation": raw_violations, + "metrics_and_subgroups": metric_violations, + "human_review_ledger": review_violations, + "geometric_leakage_audit": leakage_violations, + "vault_access_evidence": vault_violations, + } + all_violations = sorted({item for values in categories.values() for item in values}) + blockers: list[str] = [] + if model_gate.get("status") == "not_evaluable": + blockers.append( + str(model_gate.get("reason") or "Active model bytes are unavailable.") + ) + if runtime_status == "not_evaluable": + blockers.append( + str(runtime_observation.get("reason") or "CUDA runtime is unavailable.") + ) + common = sorted( + { + *model_violations, + *manifest_violations, + *artifact_violations, + *configuration_violations, + *protected_violations, + *inference_violations, + } + ) + + def derived(own: list[str], evidence: dict[str, Any]) -> dict[str, Any]: + gate_violations = sorted({*common, *own}) + status = "fail" if gate_violations else "not_evaluable" if blockers else "pass" + return { + "status": status, + "violations": gate_violations, + "blockers": blockers, + "evidence": evidence, + } + + derived_gates = { + "authoritative_reference_portfolio_available": derived( + authority_violations, authority_evidence + ), + "representative_product_subgroup_support": derived( + [*raw_violations, *metric_violations], subgroup_evidence + ), + "human_review_complete": derived(review_violations, review_evidence), + "split_independence": derived(leakage_violations, leakage_evidence), + "protected_storage_isolation": derived(vault_violations, vault_evidence), + } + status = "fail" if all_violations else "not_evaluable" if blockers else "pass" return { - "status": "fail" if violations else "pass", + "status": status, + "reason": "; ".join(blockers) if blockers else None, "path": relative.as_posix(), - "manifest_sha256": sha256(manifest_path), - "violations": sorted(violations), + "manifest_sha256": sha256(resolved_manifest), + "violations": all_violations, + "blockers": blockers, + "active_model_observation": model_gate, + "runtime_observation": runtime_observation, + "evaluation_results_sha256": evaluation.get("results_canonical_json_sha256") + if evaluation + else None, + "validation_checks": { + name: { + "status": "fail" if values else "pass", + "violations": sorted(set(values)), + } + for name, values in categories.items() + }, "checked_artifacts": checked_artifacts, + "derived_gates": derived_gates, "evidence": ( - "A non-synthetic active-model inference, protected split, authority reference, " - "raw predictions and metric report are all checksum-bound." - if not violations + "Governed product evidence was independently model/runtime-verified and metrics were recomputed in-process." + if status == "pass" else None ), } @@ -226,6 +2158,9 @@ def readiness_snapshot(repo_root: Path) -> dict[str, Any]: return { "schema_version": 1, "source_paths": { + "accuracy_status": repository_file( + repo_root, "docs/accuracy-program/status.json" + ), "phase3_full_scan": repository_file( repo_root, "artifacts/evidence/accuracy/P3/full-scan-manifest.json" ), @@ -243,84 +2178,116 @@ def readiness_snapshot(repo_root: Path) -> dict[str, Any]: }, "phase3_leakage_status": leakage.get("status"), "authority_requirements": [ - {"task": "building_validation", "zone": "flanders", "primary": "grb"}, - {"task": "building_validation", "zone": "wallonia", "primary": "picc"}, - {"task": "building_validation", "zone": "brussels", "primary": "urbis"}, - {"task": "terrain_height", "zone": "flanders", "primary": "dhmv"}, - {"task": "terrain_height", "zone": "wallonia", "primary": "spw_terrain"}, - { - "task": "north_sea_bathymetry", - "zone": "belgian_north_sea", - "primary": "mdk", - }, - { - "task": "imagery_corroboration", - "zone": "belgium", - "primary": "official_orthophoto", - "contextual": "sentinel-2", - }, + dict(item) for item in REQUIRED_AUTHORITY_REQUIREMENTS ], } +def active_model_availability_gate(active_model: dict[str, Any]) -> dict[str, Any]: + configured_path_value = active_model.get("path") + configured_hash = active_model.get("sha256") + result: dict[str, Any] = { + "configured_path": str(configured_path_value or ""), + "configured_sha256": configured_hash, + } + if not isinstance(configured_path_value, str) or not configured_path_value.strip(): + return { + **result, + "status": "fail", + "reason": "Configured active model path is missing.", + } + if not is_sha256(configured_hash): + return { + **result, + "status": "fail", + "reason": "Configured active model SHA-256 is missing or invalid.", + } + configured_path = Path(configured_path_value) + if not configured_path.is_file(): + return { + **result, + "status": "not_evaluable", + "reason": "Configured active model is not locally accessible.", + } + observed_hash = sha256(configured_path) + observed_size = configured_path.stat().st_size + result.update( + { + "observed_sha256": observed_hash, + "observed_size_bytes": observed_size, + } + ) + if observed_hash != configured_hash: + return { + **result, + "status": "fail", + "reason": "Configured active model checksum does not match the local model file.", + } + expected_size = active_model.get("size_bytes") + if ( + not isinstance(expected_size, int) + or isinstance(expected_size, bool) + or expected_size != observed_size + ): + return { + **result, + "status": "fail", + "reason": "Configured active model size does not match the local model file.", + } + return {**result, "status": "pass", "reason": None} + + def product_gate_evidence( repo_root: Path, snapshot: dict[str, Any], product_baseline_manifest: Path, + runtime_probe: Callable[[], dict[str, Any]] | None = None, ) -> dict[str, Any]: - v56 = snapshot["v56_review_and_split"] - active_model = snapshot["active_model"] or {} - configured_path = Path(str(active_model.get("path") or "")) - p3_grb = snapshot["phase3_scan"].get("grb_consistency") or {} + """Build mandatory product gates exclusively from governed artifacts.""" + + active_model = snapshot.get("active_model") or {} baseline_gate = product_baseline_manifest_gate( repo_root, product_baseline_manifest, active_model, + snapshot["authority_requirements"], + runtime_probe=runtime_probe, ) + derived = baseline_gate.get("derived_gates") or {} + fallback_status = ( + "fail" if baseline_gate.get("status") == "fail" else "not_evaluable" + ) + fallback = { + "status": fallback_status, + "reason": baseline_gate.get("reason") + or "A checksum-bound governed product artifact is unavailable.", + } + phase3_leakage = snapshot.get("phase3_leakage_status") return { - "active_model_available_and_hash_verified": { - "status": "pass" - if configured_path.is_file() - and sha256(configured_path) == active_model.get("sha256") - else "not_evaluable", - "configured_path": str(configured_path), - "configured_sha256": active_model.get("sha256"), - "reason": None - if configured_path.is_file() - else "Configured active model is not locally accessible.", - }, - "authoritative_reference_portfolio_available": { - "status": "not_evaluable", - "requirements": snapshot["authority_requirements"], - "observed_grb": p3_grb, - "reason": "A task- and zone-complete governed GRB/PICC/UrbIS/DHMV/SPW/MDK reference portfolio is not locally accessible.", - }, - "human_review_complete": { - "status": "pass" if bool(v56.get("review_complete")) else "fail", - "observed": v56.get("reviewed_sample_count"), - "required": v56.get("sample_count"), - }, - "split_independence": { - "status": "pass" if bool(v56.get("split_independence_proven")) else "fail", - "cross_split_pairs_below_2000_m": v56.get("cross_split_pairs_below_2000_m"), - }, + "active_model_available_and_hash_verified": baseline_gate.get( + "active_model_observation" + ) + or active_model_availability_gate(active_model), + "authoritative_reference_portfolio_available": derived.get( + "authoritative_reference_portfolio_available", fallback + ), + "human_review_complete": derived.get("human_review_complete", fallback), + "split_independence": derived.get("split_independence", fallback), "phase3_leakage_resolved": { "status": "pass" - if snapshot.get("phase3_leakage_status") == "pass" - else "not_evaluable", - "observed": snapshot.get("phase3_leakage_status"), - }, - "protected_storage_isolation": { - "status": "pass" - if snapshot.get("protected_test_isolation") is True - else "not_evaluable", - "reason": "A physically isolated vault, scoped credentials and immutable access log are not proven.", + if phase3_leakage == "pass" + else "not_evaluable" + if phase3_leakage is None + else "fail", + "observed": phase3_leakage, }, + "protected_storage_isolation": derived.get( + "protected_storage_isolation", fallback + ), "executed_product_incumbent_baseline": baseline_gate, - "representative_product_subgroup_support": { - "status": "not_evaluable", - "reason": "No real protected raw-prediction portfolio is available for AOI/region/context subgroup support.", - }, + "representative_product_subgroup_support": derived.get( + "representative_product_subgroup_support", fallback + ), } @@ -334,7 +2301,14 @@ def build_release_gate_report( ) -> dict[str, Any]: declared_families = {item["task"] for item in evaluation["task_inventory"]} observed_families = {item["task"] for item in evaluation["results"]} - required_split_roles = {"train", "val", "calibration", "test", "background-test"} + required_split_roles = { + "train", + "val", + "calibration", + "test", + "background-test", + "challenge", + } observed_split_roles = set(split_result["leakage"]["split_counts"]) required_raw_fields = { "references", @@ -455,51 +2429,78 @@ def build_release_gate_report( "sample_id": "background-test-pure-empty", }, } - invalid_gate_states = { - f"{family}.{name}": item.get("status") - for family, gates in (("local", local_gates), ("product", product_gates)) - for name, item in gates.items() - if item.get("status") not in GATE_STATES + gate_families = {"local": local_gates, "product": product_gates} + required_names = { + "local": LOCAL_GATE_NAMES, + "product": PRODUCT_GATE_NAMES, } - local_green = not invalid_gate_states and all( - item["status"] == "pass" for item in local_gates.values() + missing_gate_names = { + family: sorted(required_names[family] - set(gates)) + for family, gates in gate_families.items() + } + unexpected_gate_names = { + family: sorted(set(gates) - required_names[family]) + for family, gates in gate_families.items() + } + invalid_gate_states: dict[str, Any] = {} + for family, gates in gate_families.items(): + for name, item in gates.items(): + state = item.get("status") if isinstance(item, dict) else None + if state not in GATE_STATES: + invalid_gate_states[f"{family}.{name}"] = state + + def family_status(family: str) -> str: + gates = gate_families[family] + if ( + missing_gate_names[family] + or unexpected_gate_names[family] + or any(key.startswith(f"{family}.") for key in invalid_gate_states) + ): + return "fail" + states = {gates[name]["status"] for name in required_names[family]} + if "fail" in states: + return "fail" + if "not_evaluable" in states: + return "not_evaluable" + return "pass" + + local_status = family_status("local") + product_status = family_status("product") + family_states = {local_status, product_status} + overall_status = ( + "fail" + if "fail" in family_states + else "not_evaluable" + if "not_evaluable" in family_states + else "pass" ) - product_green = not invalid_gate_states and all( - item["status"] == "pass" for item in product_gates.values() - ) - if not local_green: - overall_status = "fail" - elif not product_green: - overall_status = ( - "not_evaluable" - if any(item["status"] == "not_evaluable" for item in product_gates.values()) - else "fail" - ) - else: - overall_status = "pass" return { "schema_version": 2, "gate_policy": "geointel-p4-evaluation-harness-v2", "status": overall_status, "phase_decision": "ready_for_phase5" if overall_status == "pass" else "blocked", - "local_harness_status": "pass" if local_green else "fail", - "product_benchmark_status": "pass" if product_green else "not_evaluable", + "local_harness_status": local_status, + "product_benchmark_status": product_status, "promotion_allowed": False, "numeric_model_release_targets": "not_frozen_without_reviewed_representative_incumbent_baseline", "local_gates": local_gates, "product_gates": product_gates, "invalid_gate_states": invalid_gate_states, + "missing_gate_names": missing_gate_names, + "unexpected_gate_names": unexpected_gate_names, "critical_subgroup_policy": ( "Any required subgroup with insufficient support, missing metrics, a failed " "non-inferiority comparison or regression blocks promotion; averages cannot override it." ), "decision": ( - "The local contract harness passes, but Phase 4 remains in progress and Phase 5 " - "is not ready until a real protected product incumbent baseline is evaluable." - if local_green and not product_green - else "All Phase 4 completion gates pass." - if local_green and product_green - else "The local Phase 4 harness has failing contract gates." + "All Phase 4 completion gates pass." + if overall_status == "pass" + else "At least one mandatory Phase 4 gate failed; Phase 5 remains blocked." + if overall_status == "fail" + else ( + "All evaluated Phase 4 gates pass, but mandatory product evidence remains " + "not evaluable; Phase 5 remains blocked." + ) ), } @@ -542,7 +2543,7 @@ def firewall_contract_checks( protected_item = protected["samples"][0] checks: dict[str, bool] = {} try: - assert_training_inputs_safe([], train, protected) + assert_training_inputs_safe([], train, protected, trusted_fixture_mode=True) except LeakageError: checks["clean_train_allowed"] = False else: @@ -557,7 +2558,9 @@ def firewall_contract_checks( ), ): try: - assert_training_inputs_safe(paths, records, protected) + assert_training_inputs_safe( + paths, records, protected, trusted_fixture_mode=True + ) except LeakageError: checks[name] = True else: @@ -590,6 +2593,8 @@ def metric_results_without_raw(evaluation: dict[str, Any]) -> list[dict[str, Any def build_input_manifest( repo_root: Path, snapshot: dict[str, Any], + product_baseline_manifest: Path, + product_gates: dict[str, Any], ) -> dict[str, Any]: code_paths = [ "scripts/accuracy_phase4_evaluator.py", @@ -602,11 +2607,40 @@ def build_input_manifest( "fixtures/accuracy/p4/split-source-manifest.json", "fixtures/accuracy/p4/protected-baseline-cases.json", "fixtures/golden/golden_qa_benchmarks.json", + "docs/accuracy-program/status.json", "docs/accuracy-program/05-metric-framework.md", "docs/accuracy-program/07-source-authority-matrix.md", "artifacts/evidence/accuracy/P3/full-scan-manifest.json", "artifacts/evidence/accuracy/P3/leakage-report.json", ] + baseline_gate = product_gates.get("executed_product_incumbent_baseline") or {} + manifest_binding: dict[str, Any] | None = None + resolved_manifest = product_baseline_manifest.resolve() + try: + requested_path = resolved_manifest.relative_to(repo_root.resolve()).as_posix() + except (OSError, ValueError): + requested_path = str(product_baseline_manifest) + if resolved_manifest.is_file(): + try: + relative_manifest = resolved_manifest.relative_to(repo_root.resolve()) + except (OSError, ValueError): + relative_manifest = None + if relative_manifest is not None: + manifest_binding = { + "path": relative_manifest.as_posix(), + "sha256": sha256(resolved_manifest), + "size_bytes": resolved_manifest.stat().st_size, + } + product_binding = { + "requested_path": requested_path, + "validation_status": baseline_gate.get("status"), + "manifest": manifest_binding, + "artifacts": sorted( + baseline_gate.get("checked_artifacts") or [], + key=lambda item: str(item.get("role")), + ), + "product_gate_evidence_sha256": canonical_hash(product_gates), + } return { "schema_version": 2, "repository_commit": repository_commit(repo_root), @@ -615,9 +2649,10 @@ def build_input_manifest( "readiness_snapshot": snapshot, "readiness_snapshot_sha256": canonical_hash(snapshot), "runtime": runtime_identity(), + "product_baseline": product_binding, "model_execution": { - "status": "not_evaluable", - "reason": "The configured active model and governed protected product inputs are not locally accessible.", + "status": baseline_gate.get("status", "fail"), + "reason": baseline_gate.get("reason"), "configured_active_model": snapshot.get("active_model"), }, } @@ -625,13 +2660,13 @@ def build_input_manifest( def run_workflow( repo_root: Path, - output_dir: Path, + output_dir: Path | None, product_baseline_manifest: Path | None = None, ) -> dict[str, Any]: source_path = repo_root / "fixtures/accuracy/p4/split-source-manifest.json" cases_path = repo_root / "fixtures/accuracy/p4/protected-baseline-cases.json" source = json.loads(source_path.read_text(encoding="utf-8")) - development, protected, leakage = build_manifests(source) + development, protected, leakage = build_manifests(source, trusted_fixture_mode=True) if leakage["status"] != "pass": raise LeakageError( f"Leakage gate failed with {leakage['finding_count']} findings" @@ -673,7 +2708,9 @@ def run_workflow( firewall_checks, product_gates, ) - input_manifest = build_input_manifest(repo_root, snapshot) + input_manifest = build_input_manifest( + repo_root, snapshot, baseline_path, product_gates + ) evaluation_contract = { "schema_version": 2, "benchmark_id": BENCHMARK_ID, @@ -862,6 +2899,8 @@ def run_workflow( }, "code": input_manifest["code"], "runtime": input_manifest["runtime"], + "product_baseline": input_manifest["product_baseline"], + "product_gate_evidence_sha256": canonical_hash(product_gates), "split_manifests": { "development_sha256": development["manifest_sha256"], "protected_sha256": protected["manifest_sha256"], @@ -869,7 +2908,10 @@ def run_workflow( }, "inference_and_selection": { "synthetic_reference_harness": True, - "production_model_inference_executed": False, + "production_model_inference_executed": product_gates[ + "executed_product_incumbent_baseline" + ]["status"] + == "pass", "test_used_for_selection": False, "background_test_used_for_selection": False, "challenge_labels_available": False, @@ -882,8 +2924,29 @@ def run_workflow( "reference_baseline_sha256": golden["content_sha256"], "claim_boundary": evaluation["claim_boundary"], } + run_fingerprint = canonical_hash( + { + "workflow_version": WORKFLOW_VERSION, + "input_manifest_sha256": input_manifest_file_sha256, + "product_gate_evidence_sha256": canonical_hash(product_gates), + "evaluation_results_sha256": evaluation["results_canonical_json_sha256"], + "development_split_sha256": development["manifest_sha256"], + "protected_split_sha256": protected["manifest_sha256"], + } + ) + evidence_run_id = f"p4-{WORKFLOW_VERSION}-{run_fingerprint[:20]}" + benchmark_manifest["evidence_run_id"] = evidence_run_id benchmark_manifest["manifest_sha256"] = canonical_hash(benchmark_manifest) gate_report["benchmark_manifest_sha256"] = benchmark_manifest["manifest_sha256"] + target_output_dir = output_dir or ( + repo_root / "artifacts/evidence/accuracy/P4/runs" / evidence_run_id + ) + try: + evidence_path = ( + target_output_dir.resolve().relative_to(repo_root.resolve()).as_posix() + ) + except (OSError, ValueError): + evidence_path = str(target_output_dir) workflow_summary = { "schema_version": 2, "status": gate_report["status"], @@ -891,6 +2954,9 @@ def run_workflow( "local_harness_status": gate_report["local_harness_status"], "product_benchmark_status": gate_report["product_benchmark_status"], "benchmark_manifest_sha256": benchmark_manifest["manifest_sha256"], + "evidence_run_id": evidence_run_id, + "evidence_path": evidence_path, + "product_gate_evidence_sha256": canonical_hash(product_gates), "split_counts": leakage["split_counts"], "task_family_count": evaluation["task_count"], "implemented_capability_count": len(evaluation["task_inventory"]), @@ -931,9 +2997,8 @@ def run_workflow( "workflow-summary.json": workflow_summary, } evidence = build_evidence_manifest(artifacts, benchmark_manifest) - for name, payload in sorted(artifacts.items()): - write_json_immutable(output_dir / name, payload) - write_json_immutable(output_dir / "evidence-manifest.json", evidence) + evidence_bundle = {**artifacts, "evidence-manifest.json": evidence} + write_json_bundle_immutable(target_output_dir, evidence_bundle) return workflow_summary @@ -943,7 +3008,10 @@ def parse_args() -> argparse.Namespace: parser.add_argument( "--output-dir", type=Path, - default=ROOT / "artifacts/evidence/accuracy/P4/reference-harness-v2", + default=None, + help=( + "Override the default content-addressed artifacts/evidence/accuracy/P4/runs/ directory." + ), ) parser.add_argument( "--product-baseline-manifest", @@ -969,7 +3037,7 @@ def main() -> int: try: summary = run_workflow( args.repo_root.resolve(), - args.output_dir.resolve(), + args.output_dir.resolve() if args.output_dir else None, args.product_baseline_manifest.resolve() if args.product_baseline_manifest else None,