Jens
faeb58ef6d
GeoIntel release gates / Compile, test, contracts and builds (push) Successful in 1m49s
GeoIntel release gates / Python and npm vulnerability policy (push) Successful in 21s
GeoIntel release gates / Production AI image, SBOM and container scan (push) Successful in 5m39s
GeoIntel release gates / Deploy exact gated revision to Unraid (push) Failing after 58m43s
1275 lines
47 KiB
Python
1275 lines
47 KiB
Python
from __future__ import annotations
|
|
|
|
import copy
|
|
import hashlib
|
|
import json
|
|
import sys
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
import pytest
|
|
|
|
|
|
ROOT = Path(__file__).resolve().parents[2]
|
|
SCRIPTS = ROOT / "scripts"
|
|
if str(SCRIPTS) not in sys.path:
|
|
sys.path.insert(0, str(SCRIPTS))
|
|
|
|
from accuracy_phase4_evaluator import ( # noqa: E402
|
|
TASKS,
|
|
canonical_hash,
|
|
evaluate_cases,
|
|
task_inventory,
|
|
)
|
|
from generate_accuracy_phase4_splits import ( # noqa: E402
|
|
LeakageError,
|
|
assert_training_inputs_safe,
|
|
build_manifests,
|
|
)
|
|
from run_accuracy_phase4_benchmark import ( # noqa: E402
|
|
EvidenceConflictError,
|
|
PRODUCT_GATE_NAMES,
|
|
REQUIRED_AUTHORITY_REQUIREMENTS,
|
|
REQUIRED_SUBGROUP_DIMENSION_FIELDS,
|
|
SUBGROUP_RELEASE_POLICY,
|
|
active_model_availability_gate,
|
|
build_release_gate_report,
|
|
canonical_golden_baseline,
|
|
readiness_snapshot,
|
|
firewall_contract_checks,
|
|
product_baseline_manifest_gate,
|
|
product_gate_evidence,
|
|
run_workflow,
|
|
)
|
|
|
|
|
|
SOURCE = ROOT / "fixtures/accuracy/p4/split-source-manifest.json"
|
|
CASES = ROOT / "fixtures/accuracy/p4/protected-baseline-cases.json"
|
|
|
|
|
|
def _fixture_hash(value: str) -> str:
|
|
return hashlib.sha256(value.encode("utf-8")).hexdigest()
|
|
|
|
|
|
def _write_governed_json(
|
|
repo_root: Path,
|
|
path: Path,
|
|
payload: dict[str, Any],
|
|
) -> dict[str, Any]:
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
content = (
|
|
json.dumps(payload, ensure_ascii=False, indent=2, sort_keys=True) + "\n"
|
|
).encode("utf-8")
|
|
path.write_bytes(content)
|
|
return {
|
|
"path": path.relative_to(repo_root).as_posix(),
|
|
"sha256": hashlib.sha256(content).hexdigest(),
|
|
"size_bytes": len(content),
|
|
}
|
|
|
|
|
|
def _build_governed_product_fixture(tmp_path: Path) -> dict[str, Any]:
|
|
"""Build structural governance evidence; this fixture makes no accuracy claim."""
|
|
|
|
repo_root = tmp_path / "repo"
|
|
evaluator_path = repo_root / "scripts/accuracy_phase4_evaluator.py"
|
|
evaluator_path.parent.mkdir(parents=True, exist_ok=True)
|
|
evaluator_path.write_text("# governed evaluator fixture\n", encoding="utf-8")
|
|
evaluator_hash = hashlib.sha256(evaluator_path.read_bytes()).hexdigest()
|
|
|
|
model_path = repo_root / "models/active.pt"
|
|
model_path.parent.mkdir(parents=True, exist_ok=True)
|
|
model_path.write_bytes(b"governed-model-fixture")
|
|
active_model = {
|
|
"model_id": "fixture-model",
|
|
"model_version": "1.0.0",
|
|
"path": str(model_path),
|
|
"sha256": hashlib.sha256(model_path.read_bytes()).hexdigest(),
|
|
"size_bytes": model_path.stat().st_size,
|
|
}
|
|
baseline_id = "governed-product-fixture"
|
|
evidence_root = repo_root / "artifacts/evidence/accuracy/P4/governed-fixture"
|
|
raw_path = evidence_root / "raw-predictions.json"
|
|
development_split_hash = _fixture_hash("governed-development-split-v1")
|
|
|
|
source_portfolio = json.loads(CASES.read_text(encoding="utf-8"))
|
|
templates: dict[str, dict[str, Any]] = {}
|
|
for case in source_portfolio["cases"]:
|
|
templates.setdefault(case["task"], case)
|
|
assert set(templates) == TASKS
|
|
|
|
profiles = (
|
|
{
|
|
"region": "flanders",
|
|
"municipality": "Mol",
|
|
"urbanity": "urban",
|
|
"object_size": "small",
|
|
"source": "governed-grb-orthophoto",
|
|
"sensor": "aerial-rgb",
|
|
"resolution_m": 0.25,
|
|
"season": "summer",
|
|
"date": "2025-06-15",
|
|
"vegetation": "low",
|
|
"occlusion": "none",
|
|
"difficulty": "normal",
|
|
"context": "dense_urban",
|
|
},
|
|
{
|
|
"region": "wallonia",
|
|
"municipality": "Namur",
|
|
"urbanity": "rural",
|
|
"object_size": "large",
|
|
"source": "governed-picc-orthophoto",
|
|
"sensor": "multispectral-rgb",
|
|
"resolution_m": 1.0,
|
|
"season": "winter",
|
|
"date": "2025-01-15",
|
|
"vegetation": "high",
|
|
"occlusion": "partial",
|
|
"difficulty": "hard",
|
|
"context": "rural_occluded",
|
|
},
|
|
)
|
|
authority_scopes = [
|
|
{
|
|
"task": requirement["task"],
|
|
"zone": requirement["zone"],
|
|
"authority": requirement["primary"],
|
|
}
|
|
for requirement in REQUIRED_AUTHORITY_REQUIREMENTS
|
|
]
|
|
|
|
cases: list[dict[str, Any]] = []
|
|
protected_samples: list[dict[str, Any]] = []
|
|
parameters_by_task = {
|
|
task: copy.deepcopy(templates[task]["config"]) for task in sorted(TASKS)
|
|
}
|
|
for task in sorted(TASKS):
|
|
for profile_index, profile in enumerate(profiles):
|
|
stratum = "a" if profile_index == 0 else "b"
|
|
for replicate in range(5):
|
|
sample_id = f"governed-{task}-{stratum}-{replicate}"
|
|
case = copy.deepcopy(templates[task])
|
|
case["sample_id"] = sample_id
|
|
case["split"] = (
|
|
"background-test"
|
|
if task == "object_detection" and profile_index == 1
|
|
else "test"
|
|
)
|
|
case["metadata"].update(profile)
|
|
case["metadata"].update(
|
|
{
|
|
"tile_edge": profile_index == 1,
|
|
"label_review_state": "human_reviewed_fixture",
|
|
"ood": False,
|
|
}
|
|
)
|
|
case["config"] = copy.deepcopy(parameters_by_task[task])
|
|
case["lineage"] = {
|
|
"reference": {
|
|
"source_id": f"governed:{sample_id}:reference",
|
|
"source_version": "fixture-v1",
|
|
"derivation": "structural_contract_fixture_reference",
|
|
},
|
|
"prediction": {
|
|
"source_id": f"governed:{sample_id}:prediction",
|
|
"source_version": "fixture-v1",
|
|
"derivation": "structural_contract_fixture_prediction",
|
|
},
|
|
}
|
|
cases.append(case)
|
|
reference_payload = (
|
|
case["expected_anomalies"]
|
|
if task == "geospatial_data_validation"
|
|
else case["references"]
|
|
)
|
|
subgroups = {
|
|
dimension: case["metadata"][metadata_field]
|
|
for dimension, metadata_field in (
|
|
("region", "region"),
|
|
("municipality", "municipality"),
|
|
("urbanity", "urbanity"),
|
|
("object_size", "object_size"),
|
|
("source", "source"),
|
|
("sensor", "sensor"),
|
|
("resolution", "resolution_m"),
|
|
("season", "season"),
|
|
("date", "date"),
|
|
("vegetation", "vegetation"),
|
|
("occlusion", "occlusion"),
|
|
("difficulty", "difficulty"),
|
|
("context", "context"),
|
|
)
|
|
}
|
|
protected_samples.append(
|
|
{
|
|
"sample_id": sample_id,
|
|
"split": case["split"],
|
|
"task": task,
|
|
"zone": str(profile["region"]),
|
|
"aoi_id": f"independent-aoi-{task}-{stratum}-{replicate}",
|
|
"content_sha256": _fixture_hash(f"content:{sample_id}"),
|
|
"label_sha256": canonical_hash(reference_payload),
|
|
"case_input_sha256": canonical_hash(case),
|
|
"labels_access_policy": "evaluation_only",
|
|
"subgroups": subgroups,
|
|
"authority_scopes": copy.deepcopy(authority_scopes),
|
|
}
|
|
)
|
|
|
|
challenge_id = "governed-challenge-sealed"
|
|
protected_samples.append(
|
|
{
|
|
"sample_id": challenge_id,
|
|
"split": "challenge",
|
|
"task": "object_detection",
|
|
"zone": "flanders",
|
|
"aoi_id": "independent-aoi-challenge-sealed",
|
|
"content_sha256": _fixture_hash(f"content:{challenge_id}"),
|
|
"labels_sealed": True,
|
|
"subgroups": {
|
|
dimension: profiles[0][metadata_field]
|
|
for dimension, metadata_field in (
|
|
("region", "region"),
|
|
("municipality", "municipality"),
|
|
("urbanity", "urbanity"),
|
|
("object_size", "object_size"),
|
|
("source", "source"),
|
|
("sensor", "sensor"),
|
|
("resolution", "resolution_m"),
|
|
("season", "season"),
|
|
("date", "date"),
|
|
("vegetation", "vegetation"),
|
|
("occlusion", "occlusion"),
|
|
("difficulty", "difficulty"),
|
|
("context", "context"),
|
|
)
|
|
},
|
|
"authority_scopes": copy.deepcopy(authority_scopes),
|
|
}
|
|
)
|
|
evaluation_ids = sorted(case["sample_id"] for case in cases)
|
|
all_ids = sorted(sample["sample_id"] for sample in protected_samples)
|
|
task_sample_ids = {
|
|
task: sorted(case["sample_id"] for case in cases if case["task"] == task)
|
|
for task in sorted(TASKS)
|
|
}
|
|
split_counts = {
|
|
split: sum(sample["split"] == split for sample in protected_samples)
|
|
for split in sorted({sample["split"] for sample in protected_samples})
|
|
}
|
|
protected_split = {
|
|
"schema_version": 3,
|
|
"artifact_role": "protected_evaluation_split",
|
|
"protected_policy": {
|
|
"immutable": True,
|
|
"training_allowed": False,
|
|
"threshold_selection_allowed": False,
|
|
"model_selection_allowed": False,
|
|
"iterative_error_correction_allowed": False,
|
|
"challenge_labels_accessible": False,
|
|
},
|
|
"evaluator_task_inventory_sha256": canonical_hash(task_inventory()),
|
|
"samples": protected_samples,
|
|
"split_counts": split_counts,
|
|
"sample_ids_sha256": canonical_hash(all_ids),
|
|
"evaluation_sample_ids_sha256": canonical_hash(evaluation_ids),
|
|
"evaluated_task_families": sorted(TASKS),
|
|
"task_sample_ids": task_sample_ids,
|
|
"task_sample_ids_canonical_json_sha256": canonical_hash(task_sample_ids),
|
|
}
|
|
protected_descriptor = _write_governed_json(
|
|
repo_root,
|
|
evidence_root / "protected-split.json",
|
|
protected_split,
|
|
)
|
|
|
|
sample_references = [
|
|
{
|
|
"sample_id": case["sample_id"],
|
|
"task": case["task"],
|
|
"reference_payload_sha256": canonical_hash(
|
|
case["expected_anomalies"]
|
|
if case["task"] == "geospatial_data_validation"
|
|
else case["references"]
|
|
),
|
|
"reference_lineage_sha256": canonical_hash(case["lineage"]["reference"]),
|
|
}
|
|
for case in sorted(cases, key=lambda item: item["sample_id"])
|
|
]
|
|
authority_entries = [
|
|
{
|
|
"task": requirement["task"],
|
|
"zone": requirement["zone"],
|
|
"authority": requirement["primary"],
|
|
"source_classification": "authoritative",
|
|
"source_snapshot_id": f"snapshot-{requirement['primary']}-2026",
|
|
"source_snapshot_sha256": _fixture_hash(
|
|
f"snapshot:{requirement['primary']}"
|
|
),
|
|
"sample_ids": evaluation_ids,
|
|
}
|
|
for requirement in REQUIRED_AUTHORITY_REQUIREMENTS
|
|
]
|
|
authority_portfolio = {
|
|
"schema_version": 2,
|
|
"artifact_role": "authoritative_reference_portfolio",
|
|
"portfolio_id": "governed-authority-fixture",
|
|
"protected_split_sha256": protected_descriptor["sha256"],
|
|
"entries": authority_entries,
|
|
"entries_canonical_json_sha256": canonical_hash(authority_entries),
|
|
"sample_references": sample_references,
|
|
"sample_references_canonical_json_sha256": canonical_hash(sample_references),
|
|
}
|
|
authority_descriptor = _write_governed_json(
|
|
repo_root,
|
|
evidence_root / "authoritative-reference.json",
|
|
authority_portfolio,
|
|
)
|
|
|
|
portfolio_lineage = {
|
|
"origin": "governed_product_inference",
|
|
"source_path": raw_path.relative_to(repo_root).as_posix(),
|
|
"version": baseline_id,
|
|
"active_model_sha256": active_model["sha256"],
|
|
"configuration_sha256": "0" * 64,
|
|
"protected_split_sha256": protected_descriptor["sha256"],
|
|
"authoritative_reference_sha256": authority_descriptor["sha256"],
|
|
"inference_evidence_sha256": "0" * 64,
|
|
}
|
|
raw_portfolio = {
|
|
"schema_version": 2,
|
|
"portfolio_kind": "governed_product_baseline",
|
|
"portfolio_id": baseline_id,
|
|
"split_roles": ["test", "background-test"],
|
|
"selection_policy": "frozen_validation_calibration_only_no_protected_selection",
|
|
"protected_policy": {
|
|
"operating_point_selection_allowed": False,
|
|
"diagnostic_curves_select_operating_point": False,
|
|
"test_feedback_allowed": False,
|
|
"threshold_selection_source": "pre_registered_configuration_only",
|
|
},
|
|
"portfolio_lineage": portfolio_lineage,
|
|
"claim_boundary": (
|
|
"Governed product baseline structural fixture validates evidence "
|
|
"contracts only and makes no production accuracy claim."
|
|
),
|
|
"cases": cases,
|
|
}
|
|
preliminary_path = repo_root / "preliminary-product-portfolio.json"
|
|
_write_governed_json(repo_root, preliminary_path, raw_portfolio)
|
|
preliminary_evaluation = evaluate_cases(preliminary_path, set(evaluation_ids))
|
|
preliminary_path.unlink()
|
|
subgroup_targets = {
|
|
task: {
|
|
"metric": metrics["primary_metric"]["name"],
|
|
"direction": metrics["primary_metric"]["direction"],
|
|
"threshold": metrics["primary_metric"]["value"],
|
|
}
|
|
for task, metrics in preliminary_evaluation["portfolio_metrics"].items()
|
|
}
|
|
|
|
configuration = {
|
|
"schema_version": 2,
|
|
"artifact_role": "frozen_inference_configuration",
|
|
"active_model_sha256": active_model["sha256"],
|
|
"development_split_manifest_sha256": development_split_hash,
|
|
"frozen_before_protected_access": True,
|
|
"frozen_at": "2026-08-02T09:00:00+00:00",
|
|
"protected_data_used": False,
|
|
"threshold_selection_source": "validation_and_calibration",
|
|
"parameters_by_task": parameters_by_task,
|
|
"subgroup_release_policy": SUBGROUP_RELEASE_POLICY,
|
|
"subgroup_release_targets": subgroup_targets,
|
|
"subgroup_release_targets_sha256": canonical_hash(subgroup_targets),
|
|
}
|
|
configuration_descriptor = _write_governed_json(
|
|
repo_root,
|
|
evidence_root / "configuration.json",
|
|
configuration,
|
|
)
|
|
|
|
execution_id = "cuda-execution-fixture-001"
|
|
runtime_observation = {
|
|
"status": "pass",
|
|
"device": "cuda:0",
|
|
"device_name": "NVIDIA governed fixture",
|
|
"gpu_uuid": "GPU-governed-fixture",
|
|
"driver_version": "570.00",
|
|
"cuda_runtime_version": "12.8",
|
|
"torch_version": "2.7.0",
|
|
"cuda_device_count": 1,
|
|
"kernel_execution_confirmed": True,
|
|
}
|
|
inference_evidence = {
|
|
"schema_version": 2,
|
|
"artifact_role": "governed_cuda_inference_execution",
|
|
"execution_id": execution_id,
|
|
"active_model_sha256": active_model["sha256"],
|
|
"configuration_sha256": configuration_descriptor["sha256"],
|
|
"evaluator_sha256": evaluator_hash,
|
|
"protected_split_sha256": protected_descriptor["sha256"],
|
|
"executed": True,
|
|
"exit_code": 0,
|
|
"test_used_for_selection": False,
|
|
"device_type": "cuda",
|
|
"device": "cuda:0",
|
|
"torch_cuda_is_available": True,
|
|
"cuda_device_count": 1,
|
|
"kernel_execution_confirmed": True,
|
|
"batch_failure_count": 0,
|
|
"torch_version": "2.7.0",
|
|
"cuda_runtime_version": "12.8",
|
|
"driver_version": "570.00",
|
|
"started_at": "2026-08-02T10:00:00+00:00",
|
|
"finished_at": "2026-08-02T10:05:00+00:00",
|
|
"nvidia_smi": {
|
|
"gpu_uuid": "GPU-governed-fixture",
|
|
"device_name": "NVIDIA governed fixture",
|
|
"driver_version": "570.00",
|
|
"cuda_version": "12.8",
|
|
"query_output_sha256": _fixture_hash("nvidia-smi-output"),
|
|
},
|
|
"processed_sample_ids": evaluation_ids,
|
|
"processed_sample_ids_sha256": canonical_hash(evaluation_ids),
|
|
"successful_sample_count": len(evaluation_ids),
|
|
}
|
|
inference_descriptor = _write_governed_json(
|
|
repo_root,
|
|
evidence_root / "inference-evidence.json",
|
|
inference_evidence,
|
|
)
|
|
|
|
raw_portfolio["portfolio_lineage"]["configuration_sha256"] = (
|
|
configuration_descriptor["sha256"]
|
|
)
|
|
raw_portfolio["portfolio_lineage"]["inference_evidence_sha256"] = (
|
|
inference_descriptor["sha256"]
|
|
)
|
|
raw_descriptor = _write_governed_json(repo_root, raw_path, raw_portfolio)
|
|
evaluation = evaluate_cases(raw_path, set(evaluation_ids))
|
|
|
|
metric_report = {
|
|
"schema_version": 3,
|
|
"artifact_role": "protected_metric_report",
|
|
"active_model_sha256": active_model["sha256"],
|
|
"configuration_sha256": configuration_descriptor["sha256"],
|
|
"evaluator_sha256": evaluator_hash,
|
|
"protected_split_sha256": protected_descriptor["sha256"],
|
|
"raw_predictions_sha256": raw_descriptor["sha256"],
|
|
"evaluator_version": evaluation["evaluator_version"],
|
|
"portfolio_kind": evaluation["portfolio_kind"],
|
|
"portfolio_id": evaluation["portfolio_id"],
|
|
"portfolio_file_sha256": evaluation["portfolio_file_sha256"],
|
|
"portfolio_canonical_json_sha256": evaluation[
|
|
"portfolio_canonical_json_sha256"
|
|
],
|
|
"evaluated_task_families": evaluation["evaluated_task_families"],
|
|
"task_count": evaluation["task_count"],
|
|
"case_count": evaluation["case_count"],
|
|
"task_inventory": evaluation["task_inventory"],
|
|
"task_inventory_sha256": canonical_hash(evaluation["task_inventory"]),
|
|
"results": evaluation["results"],
|
|
"results_canonical_json_sha256": canonical_hash(evaluation["results"]),
|
|
"portfolio_metrics": evaluation["portfolio_metrics"],
|
|
"portfolio_metrics_canonical_json_sha256": canonical_hash(
|
|
evaluation["portfolio_metrics"]
|
|
),
|
|
"subgroups": evaluation["subgroups"],
|
|
"subgroups_canonical_json_sha256": canonical_hash(evaluation["subgroups"]),
|
|
"failures": evaluation["failures"],
|
|
"failures_canonical_json_sha256": canonical_hash(evaluation["failures"]),
|
|
"failure_taxonomy": evaluation["failure_taxonomy"],
|
|
"failure_taxonomy_canonical_json_sha256": canonical_hash(
|
|
evaluation["failure_taxonomy"]
|
|
),
|
|
"subgroup_dimension_mapping": REQUIRED_SUBGROUP_DIMENSION_FIELDS,
|
|
"subgroup_release_policy": SUBGROUP_RELEASE_POLICY,
|
|
"pre_registered_targets": subgroup_targets,
|
|
"pre_registered_targets_sha256": canonical_hash(subgroup_targets),
|
|
}
|
|
metric_descriptor = _write_governed_json(
|
|
repo_root,
|
|
evidence_root / "metric-report.json",
|
|
metric_report,
|
|
)
|
|
|
|
review_entries = []
|
|
protected_by_id = {sample["sample_id"]: sample for sample in protected_samples}
|
|
for sample_id in evaluation_ids:
|
|
sample = protected_by_id[sample_id]
|
|
base_entry = {
|
|
"sample_id": sample_id,
|
|
"reviewer_id": "human-reviewer-fixture",
|
|
"review_timestamp": "2026-08-02T08:00:00+00:00",
|
|
"decision": "accepted",
|
|
"label_sha256": sample["label_sha256"],
|
|
"case_input_sha256": sample["case_input_sha256"],
|
|
}
|
|
review_entries.append(
|
|
{
|
|
**base_entry,
|
|
"entry_canonical_json_sha256": canonical_hash(base_entry),
|
|
}
|
|
)
|
|
review_ledger = {
|
|
"schema_version": 1,
|
|
"artifact_role": "human_review_ledger",
|
|
"protected_split_sha256": protected_descriptor["sha256"],
|
|
"raw_predictions_sha256": raw_descriptor["sha256"],
|
|
"entries": review_entries,
|
|
"entries_canonical_json_sha256": canonical_hash(review_entries),
|
|
}
|
|
review_descriptor = _write_governed_json(
|
|
repo_root,
|
|
evidence_root / "human-review-ledger.json",
|
|
review_ledger,
|
|
)
|
|
|
|
leakage_audit = {
|
|
"schema_version": 1,
|
|
"artifact_role": "geometric_leakage_audit",
|
|
"protected_split_sha256": protected_descriptor["sha256"],
|
|
"raw_predictions_sha256": raw_descriptor["sha256"],
|
|
"development_split_manifest_sha256": development_split_hash,
|
|
"distance_threshold_m": 2000.0,
|
|
"projected_crs": "EPSG:31370",
|
|
"algorithm": "projected_geometry_nearest_aoi_distance_v1",
|
|
"evaluation_sample_ids_sha256": canonical_hash(evaluation_ids),
|
|
"below_threshold_pair_count": 0,
|
|
"below_threshold_pairs": [],
|
|
"minimum_observed_distance_m": 2500.0,
|
|
}
|
|
leakage_descriptor = _write_governed_json(
|
|
repo_root,
|
|
evidence_root / "geometric-leakage-audit.json",
|
|
leakage_audit,
|
|
)
|
|
|
|
access_base = {
|
|
"sequence": 1,
|
|
"timestamp": "2026-08-02T10:00:00+00:00",
|
|
"actor": "phase4-evaluator",
|
|
"purpose": "evaluation_only",
|
|
"operation": "read",
|
|
"sample_ids": evaluation_ids,
|
|
"previous_entry_sha256": "0" * 64,
|
|
}
|
|
access_entry = {**access_base, "entry_sha256": canonical_hash(access_base)}
|
|
access_log = [access_entry]
|
|
vault_evidence = {
|
|
"schema_version": 1,
|
|
"artifact_role": "vault_access_evidence",
|
|
"protected_split_sha256": protected_descriptor["sha256"],
|
|
"raw_predictions_sha256": raw_descriptor["sha256"],
|
|
"execution_id": execution_id,
|
|
"vault_mode": "read_only_evaluation",
|
|
"access_log": access_log,
|
|
"access_log_canonical_json_sha256": canonical_hash(access_log),
|
|
"challenge_labels_accessed": False,
|
|
}
|
|
vault_descriptor = _write_governed_json(
|
|
repo_root,
|
|
evidence_root / "vault-access-evidence.json",
|
|
vault_evidence,
|
|
)
|
|
|
|
manifest = {
|
|
"schema_version": 2,
|
|
"manifest_type": "geointel_governed_product_baseline",
|
|
"baseline_id": baseline_id,
|
|
"created_at": "2026-08-02T10:06:00+00:00",
|
|
"status": "pass",
|
|
"synthetic": False,
|
|
"active_model": {
|
|
key: active_model[key]
|
|
for key in ("model_id", "model_version", "sha256", "size_bytes")
|
|
},
|
|
"active_model_sha256": active_model["sha256"],
|
|
"evaluator_sha256": evaluator_hash,
|
|
"configuration_sha256": configuration_descriptor["sha256"],
|
|
"development_split_manifest_sha256": development_split_hash,
|
|
"selection_isolation": {
|
|
"test_used_for_training": False,
|
|
"test_used_for_threshold_selection": False,
|
|
"test_used_for_model_selection": False,
|
|
"test_used_for_iterative_error_correction": False,
|
|
"challenge_labels_accessed": False,
|
|
"operating_point_frozen_before_protected_inference": True,
|
|
"configuration_sha256": configuration_descriptor["sha256"],
|
|
},
|
|
"inference": {
|
|
"executed": True,
|
|
"execution_id": execution_id,
|
|
"device": "cuda:0",
|
|
"test_used_for_selection": False,
|
|
},
|
|
"configuration": configuration_descriptor,
|
|
"protected_split_manifest": protected_descriptor,
|
|
"authoritative_reference_manifest": authority_descriptor,
|
|
"inference_evidence": inference_descriptor,
|
|
"raw_predictions": raw_descriptor,
|
|
"metric_report": metric_descriptor,
|
|
"human_review_ledger": review_descriptor,
|
|
"geometric_leakage_audit": leakage_descriptor,
|
|
"vault_access_evidence": vault_descriptor,
|
|
}
|
|
manifest_path = evidence_root / "product-baseline-manifest.json"
|
|
_write_governed_json(repo_root, manifest_path, manifest)
|
|
return {
|
|
"repo_root": repo_root,
|
|
"manifest_path": manifest_path,
|
|
"active_model": active_model,
|
|
"runtime_observation": runtime_observation,
|
|
"evaluation": evaluation,
|
|
"evaluation_ids": evaluation_ids,
|
|
}
|
|
|
|
|
|
def _rewrite_governed_artifact(
|
|
fixture: dict[str, Any],
|
|
role: str,
|
|
mutate: Any,
|
|
) -> None:
|
|
repo_root = fixture["repo_root"]
|
|
manifest_path = fixture["manifest_path"]
|
|
manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
|
|
artifact_path = repo_root / manifest[role]["path"]
|
|
payload = json.loads(artifact_path.read_text(encoding="utf-8"))
|
|
mutate(payload)
|
|
manifest[role] = _write_governed_json(repo_root, artifact_path, payload)
|
|
_write_governed_json(repo_root, manifest_path, manifest)
|
|
|
|
|
|
def load_source() -> dict:
|
|
return json.loads(SOURCE.read_text(encoding="utf-8"))
|
|
|
|
|
|
def evaluation_inputs() -> tuple[dict, dict, dict, dict]:
|
|
source = load_source()
|
|
development, protected, leakage = build_manifests(source, trusted_fixture_mode=True)
|
|
split_result = {
|
|
"development": development,
|
|
"protected": protected,
|
|
"leakage": leakage,
|
|
}
|
|
allowed = {
|
|
item["sample_id"]
|
|
for item in protected["samples"]
|
|
if item["split"] in {"test", "background-test"}
|
|
}
|
|
evaluation = evaluate_cases(CASES, allowed)
|
|
portfolio = json.loads(CASES.read_text(encoding="utf-8"))
|
|
firewall = firewall_contract_checks(split_result, CASES)
|
|
return split_result, evaluation, portfolio, firewall
|
|
|
|
|
|
def _governed_baseline_gate(fixture: dict[str, Any]) -> dict[str, Any]:
|
|
# Runtime is the only mocked part: the fixture tests evidence structure, not accuracy.
|
|
return product_baseline_manifest_gate(
|
|
fixture["repo_root"],
|
|
fixture["manifest_path"],
|
|
fixture["active_model"],
|
|
[dict(item) for item in REQUIRED_AUTHORITY_REQUIREMENTS],
|
|
runtime_probe=lambda: copy.deepcopy(fixture["runtime_observation"]),
|
|
)
|
|
|
|
|
|
def test_governed_product_baseline_validator_accepts_evaluator_derived_fixture(
|
|
tmp_path: Path,
|
|
) -> None:
|
|
fixture = _build_governed_product_fixture(tmp_path)
|
|
|
|
gate = _governed_baseline_gate(fixture)
|
|
|
|
assert gate["status"] == "pass"
|
|
assert gate["violations"] == []
|
|
assert len(gate["checked_artifacts"]) == 9
|
|
assert all(
|
|
check["status"] == "pass" for check in gate["validation_checks"].values()
|
|
)
|
|
assert fixture["evaluation"]["case_count"] == 70
|
|
assert set(fixture["evaluation"]["evaluated_task_families"]) == TASKS
|
|
assert all(
|
|
metrics["case_support"] == 10
|
|
for metrics in fixture["evaluation"]["portfolio_metrics"].values()
|
|
)
|
|
for name in (
|
|
"authoritative_reference_portfolio_available",
|
|
"human_review_complete",
|
|
"split_independence",
|
|
"protected_storage_isolation",
|
|
"representative_product_subgroup_support",
|
|
):
|
|
assert gate["derived_gates"][name]["status"] == "pass"
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("role", "mutation", "expected_violation"),
|
|
[
|
|
(
|
|
"inference_evidence",
|
|
"cuda_unavailable",
|
|
"inference_evidence:torch_cuda_unavailable",
|
|
),
|
|
(
|
|
"authoritative_reference_manifest",
|
|
"authority_missing",
|
|
"authoritative_reference:missing_requirement",
|
|
),
|
|
(
|
|
"raw_predictions",
|
|
"prediction_missing",
|
|
"raw_predictions:",
|
|
),
|
|
(
|
|
"metric_report",
|
|
"subgroup_count",
|
|
"metric_report:",
|
|
),
|
|
(
|
|
"human_review_ledger",
|
|
"review_missing",
|
|
"human_review:",
|
|
),
|
|
(
|
|
"geometric_leakage_audit",
|
|
"close_pair",
|
|
"geometric_leakage:",
|
|
),
|
|
(
|
|
"vault_access_evidence",
|
|
"training_access",
|
|
"vault_access:",
|
|
),
|
|
],
|
|
)
|
|
def test_governed_product_baseline_validator_rejects_semantic_tampering(
|
|
tmp_path: Path,
|
|
role: str,
|
|
mutation: str,
|
|
expected_violation: str,
|
|
) -> None:
|
|
fixture = _build_governed_product_fixture(tmp_path)
|
|
|
|
def mutate(payload: dict[str, Any]) -> None:
|
|
if mutation == "cuda_unavailable":
|
|
payload["torch_cuda_is_available"] = False
|
|
elif mutation == "authority_missing":
|
|
payload["entries"].pop()
|
|
payload["entries_canonical_json_sha256"] = canonical_hash(
|
|
payload["entries"]
|
|
)
|
|
elif mutation == "prediction_missing":
|
|
missing_task = sorted(TASKS)[0]
|
|
payload["cases"] = [
|
|
case for case in payload["cases"] if case["task"] != missing_task
|
|
]
|
|
elif mutation == "subgroup_count":
|
|
payload["subgroups"]["dimensions"]["region"]["strata"]["flanders"][
|
|
"case_support"
|
|
] += 1
|
|
payload["subgroups_canonical_json_sha256"] = canonical_hash(
|
|
payload["subgroups"]
|
|
)
|
|
elif mutation == "review_missing":
|
|
payload["entries"].pop()
|
|
payload["entries_canonical_json_sha256"] = canonical_hash(
|
|
payload["entries"]
|
|
)
|
|
elif mutation == "close_pair":
|
|
payload["below_threshold_pair_count"] = 1
|
|
payload["below_threshold_pairs"] = [
|
|
{
|
|
"development_sample_id": "development-neighbour",
|
|
"protected_sample_id": fixture["evaluation_ids"][0],
|
|
"distance_m": 1999.0,
|
|
}
|
|
]
|
|
payload["minimum_observed_distance_m"] = 1999.0
|
|
elif mutation == "training_access":
|
|
entry = payload["access_log"][0]
|
|
entry["purpose"] = "training"
|
|
unsigned = {
|
|
key: value for key, value in entry.items() if key != "entry_sha256"
|
|
}
|
|
entry["entry_sha256"] = canonical_hash(unsigned)
|
|
payload["access_log_canonical_json_sha256"] = canonical_hash(
|
|
payload["access_log"]
|
|
)
|
|
else: # pragma: no cover - parametrization owns this closed set
|
|
raise AssertionError(mutation)
|
|
|
|
_rewrite_governed_artifact(fixture, role, mutate)
|
|
gate = _governed_baseline_gate(fixture)
|
|
|
|
assert gate["status"] == "fail"
|
|
assert any(
|
|
violation.startswith(expected_violation) for violation in gate["violations"]
|
|
)
|
|
|
|
|
|
def test_governed_validator_rejects_impossible_metric_before_hash_comparison(
|
|
tmp_path: Path,
|
|
) -> None:
|
|
fixture = _build_governed_product_fixture(tmp_path)
|
|
|
|
def mutate(payload: dict[str, Any]) -> None:
|
|
payload["results"][0]["metrics"]["precision"] = 1.5
|
|
payload["results_canonical_json_sha256"] = canonical_hash(payload["results"])
|
|
|
|
_rewrite_governed_artifact(fixture, "metric_report", mutate)
|
|
gate = _governed_baseline_gate(fixture)
|
|
|
|
assert gate["status"] == "fail"
|
|
assert any(
|
|
violation.startswith("metric_report:")
|
|
and ("range" in violation or "impossible" in violation)
|
|
for violation in gate["violations"]
|
|
)
|
|
|
|
|
|
def test_governed_validator_rejects_all_empty_observation_support(
|
|
tmp_path: Path,
|
|
) -> None:
|
|
fixture = _build_governed_product_fixture(tmp_path)
|
|
|
|
def mutate(payload: dict[str, Any]) -> None:
|
|
for metrics in payload["portfolio_metrics"].values():
|
|
metrics["observation_support"] = {
|
|
key: 0 for key in metrics["observation_support"]
|
|
}
|
|
payload["portfolio_metrics_canonical_json_sha256"] = canonical_hash(
|
|
payload["portfolio_metrics"]
|
|
)
|
|
|
|
_rewrite_governed_artifact(fixture, "metric_report", mutate)
|
|
gate = _governed_baseline_gate(fixture)
|
|
|
|
assert gate["status"] == "fail"
|
|
assert any(
|
|
violation.startswith("metric_report:") and "empty_support" in violation
|
|
for violation in gate["violations"]
|
|
)
|
|
|
|
|
|
def test_governed_validator_rejects_missing_evaluator_task_family(
|
|
tmp_path: Path,
|
|
) -> None:
|
|
fixture = _build_governed_product_fixture(tmp_path)
|
|
missing_task = "terrain_interpretation"
|
|
|
|
def mutate(payload: dict[str, Any]) -> None:
|
|
payload["cases"] = [
|
|
case for case in payload["cases"] if case["task"] != missing_task
|
|
]
|
|
|
|
_rewrite_governed_artifact(fixture, "raw_predictions", mutate)
|
|
gate = _governed_baseline_gate(fixture)
|
|
|
|
assert gate["status"] == "fail"
|
|
assert any(
|
|
violation.startswith("raw_predictions:")
|
|
and ("task" in violation or "evaluator" in violation)
|
|
for violation in gate["violations"]
|
|
)
|
|
|
|
|
|
def test_governed_product_baseline_validator_rejects_manifest_claim_tampering(
|
|
tmp_path: Path,
|
|
) -> None:
|
|
fixture = _build_governed_product_fixture(tmp_path)
|
|
manifest = json.loads(fixture["manifest_path"].read_text(encoding="utf-8"))
|
|
manifest["synthetic"] = True
|
|
_write_governed_json(
|
|
fixture["repo_root"],
|
|
fixture["manifest_path"],
|
|
manifest,
|
|
)
|
|
|
|
gate = _governed_baseline_gate(fixture)
|
|
|
|
assert gate["status"] == "fail"
|
|
assert "manifest:synthetic_or_unspecified" in gate["violations"]
|
|
|
|
|
|
def test_active_model_checksum_mismatch_is_explicit_failure(tmp_path: Path) -> None:
|
|
model_path = tmp_path / "active.pt"
|
|
model_path.write_bytes(b"observed-model")
|
|
gate = active_model_availability_gate(
|
|
{
|
|
"path": str(model_path),
|
|
"sha256": _fixture_hash("different-model"),
|
|
"size_bytes": model_path.stat().st_size,
|
|
}
|
|
)
|
|
|
|
assert gate["status"] == "fail"
|
|
assert gate["observed_sha256"] != gate["configured_sha256"]
|
|
assert "checksum" in gate["reason"].lower()
|
|
|
|
|
|
def test_missing_product_gates_fail_closed_and_explicit_fail_has_precedence() -> None:
|
|
split_result, evaluation, portfolio, firewall = evaluation_inputs()
|
|
missing_report = build_release_gate_report(
|
|
split_result,
|
|
evaluation,
|
|
portfolio,
|
|
canonical_golden_baseline(),
|
|
firewall,
|
|
{},
|
|
)
|
|
assert missing_report["status"] == "fail"
|
|
assert missing_report["product_benchmark_status"] == "fail"
|
|
assert set(missing_report["missing_gate_names"]["product"]) == PRODUCT_GATE_NAMES
|
|
|
|
product_gates = {name: {"status": "pass"} for name in PRODUCT_GATE_NAMES}
|
|
product_gates["executed_product_incumbent_baseline"] = {"status": "not_evaluable"}
|
|
product_gates["human_review_complete"] = {"status": "fail"}
|
|
precedence_report = build_release_gate_report(
|
|
split_result,
|
|
evaluation,
|
|
portfolio,
|
|
canonical_golden_baseline(),
|
|
firewall,
|
|
product_gates,
|
|
)
|
|
assert precedence_report["status"] == "fail"
|
|
assert precedence_report["product_benchmark_status"] == "fail"
|
|
|
|
|
|
def test_document_status_booleans_cannot_spoof_review_split_or_vault(
|
|
tmp_path: Path,
|
|
) -> None:
|
|
fixture = _build_governed_product_fixture(tmp_path)
|
|
snapshot = {
|
|
"active_model": fixture["active_model"],
|
|
"authority_requirements": [
|
|
dict(item) for item in REQUIRED_AUTHORITY_REQUIREMENTS
|
|
],
|
|
"v56_review_and_split": {
|
|
"review_complete": True,
|
|
"reviewed_sample_count": 999,
|
|
"sample_count": 999,
|
|
"split_independence_proven": True,
|
|
"cross_split_pairs_below_2000_m": 0,
|
|
},
|
|
"protected_test_isolation": True,
|
|
"phase3_leakage_status": "pass",
|
|
}
|
|
missing_manifest = (
|
|
fixture["repo_root"]
|
|
/ "artifacts/evidence/accuracy/P4/missing/product-baseline-manifest.json"
|
|
)
|
|
|
|
gates = product_gate_evidence(fixture["repo_root"], snapshot, missing_manifest)
|
|
|
|
for name in (
|
|
"human_review_complete",
|
|
"split_independence",
|
|
"protected_storage_isolation",
|
|
):
|
|
assert gates[name]["status"] in {"fail", "not_evaluable"}
|
|
assert gates[name]["status"] != "pass"
|
|
|
|
|
|
def test_all_mandatory_product_gates_make_phase5_reachable() -> None:
|
|
split_result, evaluation, portfolio, firewall = evaluation_inputs()
|
|
product_gates = {name: {"status": "pass"} for name in PRODUCT_GATE_NAMES}
|
|
|
|
report = build_release_gate_report(
|
|
split_result,
|
|
evaluation,
|
|
portfolio,
|
|
canonical_golden_baseline(),
|
|
firewall,
|
|
product_gates,
|
|
)
|
|
|
|
assert report["status"] == "pass"
|
|
assert report["phase_decision"] == "ready_for_phase5"
|
|
assert report["missing_gate_names"] == {"local": [], "product": []}
|
|
assert set(
|
|
report["local_gates"]["normative_split_roles_and_leakage"]["required_roles"]
|
|
) == {"train", "val", "calibration", "test", "background-test", "challenge"}
|
|
|
|
|
|
def test_split_fixture_is_order_independent_and_has_all_roles() -> None:
|
|
source = load_source()
|
|
development, protected, leakage = build_manifests(source, trusted_fixture_mode=True)
|
|
reversed_source = copy.deepcopy(source)
|
|
reversed_source["samples"].reverse()
|
|
reversed_development, reversed_protected, reversed_leakage = build_manifests(
|
|
reversed_source, trusted_fixture_mode=True
|
|
)
|
|
|
|
assert leakage["status"] == "pass"
|
|
assert leakage["finding_count"] == 0
|
|
assert leakage["split_counts"] == {
|
|
"background-test": 2,
|
|
"calibration": 2,
|
|
"challenge": 4,
|
|
"test": 7,
|
|
"train": 3,
|
|
"val": 3,
|
|
}
|
|
assert reversed_development["manifest_sha256"] == development["manifest_sha256"]
|
|
assert reversed_protected["manifest_sha256"] == protected["manifest_sha256"]
|
|
assert reversed_leakage == leakage
|
|
|
|
|
|
def test_training_firewall_rejects_non_train_and_protected_lineage() -> None:
|
|
development, protected, leakage = build_manifests(
|
|
load_source(), trusted_fixture_mode=True
|
|
)
|
|
assert leakage["status"] == "pass"
|
|
train = [item for item in development["samples"] if item["split"] == "train"]
|
|
validation = next(item for item in development["samples"] if item["split"] == "val")
|
|
|
|
assert_training_inputs_safe([], train, protected, trusted_fixture_mode=True)
|
|
with pytest.raises(LeakageError, match="non_train_role"):
|
|
assert_training_inputs_safe(
|
|
[], [validation], protected, trusted_fixture_mode=True
|
|
)
|
|
with pytest.raises(LeakageError, match="protected_identity"):
|
|
disguised = dict(train[0])
|
|
disguised["source_family"] = protected["samples"][0]["source_family"]
|
|
assert_training_inputs_safe(
|
|
[], [disguised], protected, trusted_fixture_mode=True
|
|
)
|
|
with pytest.raises(LeakageError, match="protected_path"):
|
|
assert_training_inputs_safe([CASES], [], protected, trusted_fixture_mode=True)
|
|
|
|
|
|
def test_task_evaluator_retains_exact_raw_inputs_metrics_and_failures() -> None:
|
|
_split_result, report, _portfolio, _firewall = evaluation_inputs()
|
|
|
|
assert report["task_count"] == 7
|
|
assert report["case_count"] == 9
|
|
assert len(report["task_inventory"]) >= 15
|
|
assert len(report["failures"]) >= 11
|
|
assert report["subgroups"]["overall_status"] == "not_evaluable"
|
|
assert all(
|
|
{
|
|
"references",
|
|
"predictions_pre_filter",
|
|
"predictions_post_filter",
|
|
"config",
|
|
"input_lineage",
|
|
"portfolio_lineage",
|
|
}
|
|
<= set(item["raw"])
|
|
for item in report["results"]
|
|
)
|
|
detection = next(
|
|
item
|
|
for item in report["results"]
|
|
if item["sample_id"] == "det-test-flanders-urban"
|
|
)
|
|
assert len(detection["raw"]["predictions_pre_filter"]) == 4
|
|
assert len(detection["raw"]["predictions_post_filter"]) == 3
|
|
assert detection["metrics"]["true_positive"] == 2
|
|
assert detection["metrics"]["false_positive"] == 1
|
|
assert detection["metrics"]["ap50"] is not None
|
|
empty = next(
|
|
item
|
|
for item in report["results"]
|
|
if item["sample_id"] == "background-test-pure-empty"
|
|
)
|
|
assert empty["metrics"]["precision"] is None
|
|
assert empty["metrics"]["recall"] is None
|
|
assert empty["metrics"]["f1"] is None
|
|
|
|
|
|
def test_product_prerequisites_cannot_pass_without_executed_baseline() -> None:
|
|
split_result, evaluation, portfolio, firewall = evaluation_inputs()
|
|
product_gates = {
|
|
"active_model_available_and_hash_verified": {"status": "pass"},
|
|
"authoritative_reference_portfolio_available": {"status": "pass"},
|
|
"human_review_complete": {"status": "pass"},
|
|
"split_independence": {"status": "pass"},
|
|
"phase3_leakage_resolved": {"status": "pass"},
|
|
"protected_storage_isolation": {"status": "pass"},
|
|
"executed_product_incumbent_baseline": {
|
|
"status": "not_evaluable",
|
|
"reason": "no raw active-model inference",
|
|
},
|
|
"representative_product_subgroup_support": {"status": "pass"},
|
|
}
|
|
report = build_release_gate_report(
|
|
split_result,
|
|
evaluation,
|
|
portfolio,
|
|
canonical_golden_baseline(),
|
|
firewall,
|
|
product_gates,
|
|
)
|
|
|
|
assert report["local_harness_status"] == "pass"
|
|
assert report["product_benchmark_status"] == "not_evaluable"
|
|
assert report["status"] == "not_evaluable"
|
|
assert report["phase_decision"] == "blocked"
|
|
|
|
|
|
def test_one_workflow_is_byte_reproducible_complete_and_fail_closed(
|
|
tmp_path: Path,
|
|
) -> None:
|
|
output = tmp_path / "p4"
|
|
first = run_workflow(ROOT, output)
|
|
first_bytes = {path.name: path.read_bytes() for path in output.glob("*.json")}
|
|
second = run_workflow(ROOT, output)
|
|
second_bytes = {path.name: path.read_bytes() for path in output.glob("*.json")}
|
|
|
|
assert first == second
|
|
assert first_bytes == second_bytes
|
|
assert first["local_harness_status"] == "pass"
|
|
assert first["product_benchmark_status"] == "fail"
|
|
assert first["status"] == "fail"
|
|
assert first["phase4_done"] is False
|
|
assert first["phase5_ready"] is False
|
|
required = {
|
|
"acceptance-gates.json",
|
|
"aoi-metrics.json",
|
|
"baseline-raw-predictions.json",
|
|
"benchmark-manifest.json",
|
|
"calibration-metrics.json",
|
|
"candidate-vs-incumbent.json",
|
|
"development-split-manifest.json",
|
|
"error-taxonomy.json",
|
|
"evaluation-contract.json",
|
|
"failure-gallery.json",
|
|
"generation-status.json",
|
|
"human-review-summary.json",
|
|
"input-manifest.json",
|
|
"latency-and-reliability.json",
|
|
"leakage-gate-report.json",
|
|
"metric-report.json",
|
|
"object-metrics.json",
|
|
"protected-split-manifest.json",
|
|
"reference-implementation-baseline.json",
|
|
"release-gate-report.json",
|
|
"split-and-leakage-audit.json",
|
|
"stratified-metrics.json",
|
|
"tile-metrics.json",
|
|
"workflow-summary.json",
|
|
"evidence-manifest.json",
|
|
}
|
|
assert required == set(first_bytes)
|
|
manifest = json.loads(
|
|
(output / "evidence-manifest.json").read_text(encoding="utf-8")
|
|
)
|
|
assert manifest["artifact_count"] == len(required) - 1
|
|
for item in manifest["artifacts"]:
|
|
path = output / item["path"]
|
|
assert path.stat().st_size == item["size_bytes"]
|
|
assert hashlib.sha256(path.read_bytes()).hexdigest() == item["sha256"]
|
|
gates = json.loads(
|
|
(output / "release-gate-report.json").read_text(encoding="utf-8")
|
|
)
|
|
assert gates["promotion_allowed"] is False
|
|
assert all(item["status"] == "pass" for item in gates["local_gates"].values())
|
|
assert {item["status"] for item in gates["product_gates"].values()} <= {
|
|
"pass",
|
|
"fail",
|
|
"not_evaluable",
|
|
}
|
|
|
|
input_manifest = json.loads(
|
|
(output / "input-manifest.json").read_text(encoding="utf-8")
|
|
)
|
|
benchmark_manifest = json.loads(
|
|
(output / "benchmark-manifest.json").read_text(encoding="utf-8")
|
|
)
|
|
assert input_manifest["product_baseline"]["validation_status"] == "not_evaluable"
|
|
assert input_manifest["product_baseline"]["artifacts"] == []
|
|
assert "fixtures/accuracy/readiness/status.json" not in {
|
|
item["path"] for item in input_manifest["inputs"]
|
|
}
|
|
assert input_manifest["readiness_snapshot"]["source_paths"][
|
|
"accuracy_status_projection"
|
|
]["selected_json_pointers"] == ["/runtime/active_model"]
|
|
assert benchmark_manifest["product_baseline"] == input_manifest["product_baseline"]
|
|
assert (
|
|
benchmark_manifest["product_gate_evidence_sha256"]
|
|
== first["product_gate_evidence_sha256"]
|
|
)
|
|
assert first["evidence_run_id"].startswith("p4-2.0.1-")
|
|
|
|
|
|
def test_readiness_snapshot_ignores_phase4_bookkeeping_but_binds_active_model(
|
|
tmp_path: Path,
|
|
) -> None:
|
|
status_path = tmp_path / "fixtures/accuracy/readiness/status.json"
|
|
scan_path = tmp_path / "fixtures/accuracy/readiness/full-scan-manifest.json"
|
|
leakage_path = tmp_path / "fixtures/accuracy/readiness/leakage-report.json"
|
|
status_path.parent.mkdir(parents=True)
|
|
scan_path.parent.mkdir(parents=True, exist_ok=True)
|
|
status = {
|
|
"generated_at": "2026-08-02T00:00:00+02:00",
|
|
"documents": ["old.md"],
|
|
"phase5": {"status": "not_ready"},
|
|
"verification": {"phase4_evaluation": {"status": "pending"}},
|
|
"phase4": {"status": "ready"},
|
|
"runtime": {
|
|
"active_model": {
|
|
"model_id": "model-a",
|
|
"path": "/models/a.pt",
|
|
"sha256": "a" * 64,
|
|
}
|
|
},
|
|
}
|
|
status_path.write_text(json.dumps(status), encoding="utf-8")
|
|
scan_path.write_text(
|
|
json.dumps({"scan_id": "scan-a", "content_hash": "b" * 64}),
|
|
encoding="utf-8",
|
|
)
|
|
leakage_path.write_text(json.dumps({"status": "attention"}), encoding="utf-8")
|
|
|
|
original = readiness_snapshot(tmp_path)
|
|
status["phase4"] = {"status": "in_progress", "evidence_run_id": "run-a"}
|
|
status["generated_at"] = "2026-08-02T05:00:00+02:00"
|
|
status["documents"] = ["old.md", "new.md"]
|
|
status["phase5"] = {"status": "blocked"}
|
|
status["verification"] = {"phase4_evaluation": {"status": "local_pass"}}
|
|
status_path.write_text(json.dumps(status), encoding="utf-8")
|
|
bookkeeping_update = readiness_snapshot(tmp_path)
|
|
|
|
assert bookkeeping_update == original
|
|
assert "accuracy_status" not in original["source_paths"]
|
|
assert original["source_paths"]["accuracy_status_projection"][
|
|
"selected_json_pointers"
|
|
] == ["/runtime/active_model"]
|
|
|
|
status["runtime"]["active_model"]["sha256"] = "c" * 64
|
|
status_path.write_text(json.dumps(status), encoding="utf-8")
|
|
assert readiness_snapshot(tmp_path) != original
|
|
|
|
assert original["source_paths"]["accuracy_status_projection"][
|
|
"sha256"
|
|
] == canonical_hash(
|
|
{
|
|
"schema_version": 1,
|
|
"runtime": {"active_model": original["active_model"]},
|
|
}
|
|
)
|
|
|
|
|
|
def test_immutable_workflow_refuses_to_replace_changed_evidence(tmp_path: Path) -> None:
|
|
output = tmp_path / "p4"
|
|
run_workflow(ROOT, output)
|
|
(output / "workflow-summary.json").write_text("{}\n", encoding="utf-8")
|
|
|
|
with pytest.raises(EvidenceConflictError, match="Refusing to overwrite"):
|
|
run_workflow(ROOT, output)
|
|
|
|
|
|
def test_immutable_workflow_rejects_rogue_nested_evidence(tmp_path: Path) -> None:
|
|
output = tmp_path / "p4"
|
|
run_workflow(ROOT, output)
|
|
rogue = output / "rogue" / "unmanifested.json"
|
|
rogue.parent.mkdir()
|
|
rogue.write_text("{}\n", encoding="utf-8")
|
|
|
|
with pytest.raises(EvidenceConflictError, match="(?i)unexpected|immutable"):
|
|
run_workflow(ROOT, output)
|