Initial public ModelForge release

scripts/bootstrap.py

@@ -0,0 +1,244 @@
+"""Bring a ModelForge installation from an empty database to a serving control plane.
+
+    python scripts/bootstrap.py --database-url postgresql+psycopg://...
+
+The whole point is that this is safe to run twice. Every step either creates what is missing or
+confirms what is already there, and the report says which of the two happened, so an operator can
+re-run it after a failure without wondering what state they are in.
+
+It never invents an operator credential. Secrets are generated by the operator and supplied through
+configuration; bootstrap verifies one is present and refuses to continue in production without it,
+because a platform that mints its own admin secret has no way to tell you it did.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+import time
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+from alembic import command
+from alembic.config import Config
+from sqlalchemy import create_engine, inspect, text
+from sqlalchemy.orm import Session
+
+ROOT = Path(__file__).resolve().parents[1]
+sys.path.insert(0, str(ROOT / "backend" / "src"))
+
+from modelforge_api.domain.release import (  # noqa: E402
+    PRODUCT_NAME,
+    PRODUCT_VERSION,
+    TARGET_SCHEMA_REVISION,
+    Compatibility,
+    schema_compatibility,
+)
+from modelforge_api.services.lifecycle import LifecycleService  # noqa: E402
+from modelforge_api.services.manifest_registry import ManifestRegistry  # noqa: E402
+from modelforge_api.services.migration_engine import MigrationEngineService  # noqa: E402
+from modelforge_api.services.observability import ObservabilityService  # noqa: E402
+from modelforge_api.services.project_registry import sync_project_registry  # noqa: E402
+from modelforge_api.services.recovery import RecoveryService  # noqa: E402
+from modelforge_api.services.registry import seed_candidate_registry  # noqa: E402
+from modelforge_api.settings import Settings  # noqa: E402
+
+
+@dataclass
+class Step:
+    name: str
+    outcome: str
+    detail: str
+    seconds: float = 0.0
+
+
+@dataclass
+class BootstrapReport:
+    steps: list[Step] = field(default_factory=list)
+    started_at: float = field(default_factory=time.time)
+
+    def record(self, name: str, outcome: str, detail: str, seconds: float = 0.0) -> None:
+        self.steps.append(Step(name, outcome, detail, round(seconds, 3)))
+
+    def as_dict(self) -> dict[str, Any]:
+        return {
+            "product": PRODUCT_NAME,
+            "version": PRODUCT_VERSION,
+            "total_seconds": round(time.time() - self.started_at, 3),
+            "steps": [
+                {
+                    "name": step.name,
+                    "outcome": step.outcome,
+                    "detail": step.detail,
+                    "seconds": step.seconds,
+                }
+                for step in self.steps
+            ],
+        }
+
+
+def wait_for_database(url: str, report: BootstrapReport, timeout: float = 120.0) -> None:
+    started = time.time()
+    engine = create_engine(url, pool_pre_ping=True)
+    last: str = ""
+    while time.time() - started < timeout:
+        try:
+            with engine.connect() as connection:
+                connection.exec_driver_sql("select 1")
+            report.record(
+                "database reachable", "OK", "accepted a connection", time.time() - started
+            )
+            engine.dispose()
+            return
+        except Exception as error:  # noqa: BLE001 - any failure means not ready yet
+            last = type(error).__name__
+            time.sleep(2)
+    engine.dispose()
+    raise SystemExit(f"the database never became reachable within {timeout:.0f}s (last: {last})")
+
+
+def migrate(url: str, report: BootstrapReport) -> str:
+    started = time.time()
+    engine = create_engine(url)
+    with engine.connect() as connection:
+        had_schema = inspect(connection).has_table("alembic_version")
+        before = (
+            connection.execute(text("select version_num from alembic_version")).scalar_one_or_none()
+            if had_schema
+            else None
+        )
+    config = Config(str(ROOT / "backend" / "alembic.ini"))
+    config.set_main_option("script_location", str(ROOT / "backend" / "alembic"))
+    config.set_main_option("sqlalchemy.url", url)
+    command.upgrade(config, "head")
+    with engine.connect() as connection:
+        after = connection.execute(text("select version_num from alembic_version")).scalar_one()
+    engine.dispose()
+    outcome = "ALREADY_CURRENT" if before == after else ("CREATED" if before is None else "UPGRADED")
+    report.record("migrations", outcome, f"{before or '(empty)'} -> {after}", time.time() - started)
+    if after != TARGET_SCHEMA_REVISION:
+        raise SystemExit(
+            f"migrations landed on {after}, but this release targets {TARGET_SCHEMA_REVISION}"
+        )
+    return str(after)
+
+
+def seed(url: str, settings: Settings, report: BootstrapReport) -> None:
+    """Every seed is an ensure_*: running it twice must not produce a second copy of anything."""
+
+    engine = create_engine(url)
+    manifests = ManifestRegistry(settings.config_root)
+    with Session(engine) as session:
+        started = time.time()
+        candidates = seed_candidate_registry(session, manifests)
+        session.commit()
+        report.record(
+            "candidate registry",
+            "SEEDED" if candidates else "ALREADY_PRESENT",
+            f"{candidates} candidate(s) added",
+            time.time() - started,
+        )
+
+        started = time.time()
+        sync = sync_project_registry(session, manifests)
+        session.commit()
+        detail = f"{len(sync.unavailable_contracts)} binding(s) waiting for a contract"
+        report.record("project registry", "SYNCED", detail, time.time() - started)
+
+        for name, service in (
+            ("lifecycle policies", LifecycleService(session)),
+            ("migration policies", MigrationEngineService(session)),
+            ("observability rules", ObservabilityService(session, settings)),
+            (
+                "recovery policies",
+                RecoveryService(session, settings, "control_plane", "bootstrap"),
+            ),
+        ):
+            started = time.time()
+            service.ensure_defaults()
+            session.commit()
+            report.record(name, "ENSURED", "defaults present", time.time() - started)
+    engine.dispose()
+
+
+def verify(url: str, settings: Settings, report: BootstrapReport) -> None:
+    engine = create_engine(url)
+    with engine.connect() as connection:
+        revision = connection.execute(text("select version_num from alembic_version")).scalar_one()
+        tables = int(
+            connection.execute(
+                text(
+                    "select count(*) from information_schema.tables where table_schema='public'"
+                )
+            ).scalar_one()
+        )
+        duplicates = connection.execute(
+            text(
+                "select count(*) from (select capability_contract_id from capability_deployments "
+                "where status='stable' group by capability_contract_id having count(*) > 1) as d"
+            )
+        ).scalar_one()
+    engine.dispose()
+    compatibility = schema_compatibility(revision)
+    report.record(
+        "schema",
+        "OK" if compatibility is Compatibility.COMPATIBLE else "INCOMPATIBLE",
+        f"revision {revision}, {tables} tables",
+    )
+    report.record(
+        "no duplicate stable identity",
+        "OK" if duplicates == 0 else "VIOLATED",
+        f"{duplicates} contract(s) with more than one stable deployment",
+    )
+    has_operator_key = settings.operator_api_key is not None
+    report.record(
+        "operator credential",
+        "PRESENT" if has_operator_key else "MISSING",
+        "supplied through configuration"
+        if has_operator_key
+        else "set MODELFORGE_OPERATOR_API_KEY before serving",
+    )
+    if not has_operator_key and settings.env == "production":
+        raise SystemExit(
+            "refusing to finish: production requires MODELFORGE_OPERATOR_API_KEY. ModelForge "
+            "never mints its own admin secret, because it would have no way to tell you it did."
+        )
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--database-url", default=None)
+    parser.add_argument("--env-file", default=None)
+    parser.add_argument("--report", default=None)
+    parser.add_argument("--skip-seed", action="store_true")
+    args = parser.parse_args(argv)
+
+    settings = (
+        Settings(_env_file=args.env_file)  # type: ignore[call-arg]
+        if args.env_file
+        else Settings()
+    )
+    url = args.database_url or settings.database_url
+
+    report = BootstrapReport()
+    print(f"{PRODUCT_NAME} {PRODUCT_VERSION} — bootstrap", flush=True)
+    wait_for_database(url, report)
+    migrate(url, report)
+    if not args.skip_seed:
+        seed(url, settings, report)
+    verify(url, settings, report)
+
+    for step in report.steps:
+        print(f"  {step.outcome:16} {step.name:28} {step.detail} ({step.seconds:.2f}s)", flush=True)
+    print(f"  bootstrap completed in {report.as_dict()['total_seconds']:.2f}s", flush=True)
+
+    if args.report:
+        Path(args.report).write_text(json.dumps(report.as_dict(), indent=2), encoding="utf-8")
+        print(f"  report written to {args.report}", flush=True)
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/check.ps1

@@ -0,0 +1,55 @@
+$ErrorActionPreference = "Stop"
+$PSNativeCommandUseErrorActionPreference = $true
+
+Push-Location backend
+try {
+  python -m compileall -q src tests
+  python -m ruff check src tests
+  python -m mypy src
+  python -m pytest -q
+} finally {
+  Pop-Location
+}
+
+Push-Location frontend
+try {
+  npm test
+  npm run build
+} finally {
+  Pop-Location
+}
+
+Push-Location node-agent
+try {
+  python -m compileall -q src tests
+  python -m ruff check src tests
+  python -m mypy src
+  python -m pytest -q
+} finally {
+  Pop-Location
+}
+
+Push-Location runtime-worker
+try {
+  python -m compileall -q src tests
+  python -m ruff check src tests
+  python -m mypy src
+  python -m pytest -q
+} finally {
+  Pop-Location
+}
+
+docker compose --env-file .env.example config --quiet
+docker compose --env-file .env.example -f docker-compose.yml -f docker-compose.gpu.yml config --quiet
+docker compose --env-file .env.example -f docker-compose.node-agent.yml config --quiet
+docker compose --env-file .env.example -f docker-compose.node-agent.yml -f docker-compose.runtime-worker.yml config --quiet
+$env:MODELFORGE_AGENT_CA_CERT_PATH = "./validation-ca.crt"
+$env:MODELFORGE_AGENT_CONTROL_PLANE_HOST_ADDRESS = "192.0.2.10"
+try {
+  docker compose --env-file .env.example -f docker-compose.node-agent.yml -f docker-compose.node-agent.private-ca.yml config --quiet
+  docker compose --env-file .env.example -f docker-compose.node-agent.yml -f docker-compose.node-agent.private-ca.yml -f docker-compose.runtime-worker.yml -f docker-compose.runtime-worker.private-ca.yml config --quiet
+} finally {
+  Remove-Item Env:MODELFORGE_AGENT_CA_CERT_PATH
+  Remove-Item Env:MODELFORGE_AGENT_CONTROL_PLANE_HOST_ADDRESS
+}
+Write-Host "M5 checks passed."

scripts/check.sh

@@ -0,0 +1,42 @@
+#!/usr/bin/env sh
+set -eu
+
+(cd backend && python -m compileall -q src tests)
+(cd backend && python -m ruff check src tests)
+(cd backend && python -m mypy src)
+(cd backend && python -m pytest -q)
+(cd node-agent && python -m compileall -q src tests)
+(cd node-agent && python -m ruff check src tests)
+(cd node-agent && python -m mypy src)
+(cd node-agent && python -m pytest -q)
+(cd runtime-worker && python -m compileall -q src tests)
+(cd runtime-worker && python -m ruff check src tests)
+(cd runtime-worker && python -m mypy src)
+(cd runtime-worker && python -m pytest -q)
+(cd frontend && npm test -- --run)
+(cd frontend && npm run build)
+# The console's API origin is compiled into the bundle, so a release image can be wrong while every
+# source-level check is green. Only assert this when a candidate image already exists locally.
+if docker image inspect "modelforge-web:$(cat VERSION)" >/dev/null 2>&1; then
+  python scripts/release_image_acceptance.py \
+    --image "modelforge-web:$(cat VERSION)" \
+    --expect-origin "${MODELFORGE_PUBLIC_API_ORIGIN:-http://localhost:8000}" \
+    --version "$(cat VERSION)"
+else
+  printf '%s\n' "console release-image acceptance skipped: no modelforge-web:$(cat VERSION) built"
+fi
+docker compose --env-file .env.example config --quiet
+docker compose --env-file .env.example -f docker-compose.yml -f docker-compose.gpu.yml config --quiet
+docker compose --env-file .env.example -f docker-compose.node-agent.yml config --quiet
+docker compose --env-file .env.example -f docker-compose.node-agent.yml \
+  -f docker-compose.runtime-worker.yml config --quiet
+MODELFORGE_AGENT_CA_CERT_PATH=./validation-ca.crt \
+MODELFORGE_AGENT_CONTROL_PLANE_HOST_ADDRESS=192.0.2.10 \
+  docker compose --env-file .env.example -f docker-compose.node-agent.yml \
+  -f docker-compose.node-agent.private-ca.yml config --quiet
+MODELFORGE_AGENT_CA_CERT_PATH=./validation-ca.crt \
+MODELFORGE_AGENT_CONTROL_PLANE_HOST_ADDRESS=192.0.2.10 \
+  docker compose --env-file .env.example -f docker-compose.node-agent.yml \
+  -f docker-compose.node-agent.private-ca.yml -f docker-compose.runtime-worker.yml \
+  -f docker-compose.runtime-worker.private-ca.yml config --quiet
+printf '%s\n' 'M5 checks passed.'

scripts/export-public-source.mjs

@@ -0,0 +1,214 @@
+#!/usr/bin/env node
+
+import crypto from "node:crypto";
+import fs from "node:fs";
+import path from "node:path";
+import process from "node:process";
+import { execFileSync } from "node:child_process";
+
+function fail(message) {
+  console.error(message);
+  process.exit(1);
+}
+
+function argument(name) {
+  const index = process.argv.indexOf(name);
+  return index >= 0 ? process.argv[index + 1] : undefined;
+}
+
+const root = fs.realpathSync(argument("--repository") ?? process.cwd());
+const outputArgument = argument("--output");
+const reportArgument = argument("--report");
+const allowlistPath = path.resolve(root, argument("--allowlist") ?? "public-source.allowlist");
+if (!outputArgument) fail("Usage: node scripts/export-public-source.mjs --output <new-directory> [--report <file>]");
+
+const output = path.resolve(outputArgument);
+const reportPath = reportArgument ? path.resolve(reportArgument) : undefined;
+if (output === root || output.startsWith(`${root}${path.sep}`)) {
+  fail("The public export must be outside the canonical repository.");
+}
+if (fs.existsSync(output)) fail(`Output already exists: ${output}`);
+const dirty = execFileSync("git", ["-C", root, "status", "--porcelain"], { encoding: "utf8" });
+if (dirty.trim()) {
+  fail("Public export blocked: commit the exact source tree before exporting it.");
+}
+const licensePath = path.join(root, "LICENSE");
+if (!fs.existsSync(licensePath)) fail("Public export blocked: canonical LICENSE is missing.");
+const licenseDigest = crypto.createHash("sha256").update(fs.readFileSync(licensePath)).digest("hex");
+if (licenseDigest !== "0d96a4ff68ad6d4b6f1f30f713b18d5184912ba8dd389f86aa7710db079abcb0") {
+  fail("Public export blocked: LICENSE is not the approved canonical AGPL-3.0 text.");
+}
+
+const allowlist = fs.readFileSync(allowlistPath, "utf8")
+  .split(/\r?\n/u)
+  .map((line) => line.trim())
+  .filter((line) => line && !line.startsWith("#"));
+
+function selected(relativePath) {
+  return allowlist.some((rule) =>
+    rule.endsWith("/**") ? relativePath.startsWith(rule.slice(0, -2)) : relativePath === rule
+  );
+}
+
+const deniedPrefixes = [
+  ".agents/", ".claude/", ".codex/", "artifacts/", "docs/quality/",
+  "reports/"
+];
+
+// The canonical source contains useful integration examples and operational tests tied to the
+// private deployment. The public root keeps the behavior while replacing those identifiers with
+// stable documentation-only examples. Transform before scanning and before hashing the export.
+const replacements = [
+  [/ssh:\/\/git@192\.168\.10\.150:222\/Jens\/ITWorx-ModelForge\.git/gu, "https://git.example.com/example/modelforge.git"],
+  [/https:\/\/gitea\.itworx\.tech\/Jens\/ITWorx-ModelForge/gu, "https://git.example.com/example/modelforge"],
+  [/git@gitea\.itworx\.tech:Jens\/ExampleRAG\.git/gu, "git@git.example.com:example/example-rag.git"],
+  [/modelforge\.itworx\.tech/giu, "modelforge.example.com"],
+  [/gitea\.itworx\.tech/giu, "git.example.com"],
+  [/192\.168\.10\.150/gu, "192.0.2.10"],
+  [/192\.168\.10\.241/gu, "192.0.2.11"],
+  [/192\.168\.10\.3/gu, "192.0.2.53"],
+  [/\bRAGCORE\b/gu, "EXAMPLE_RAG"],
+  [/\bRAGcore\b/gu, "ExampleRAG"],
+  [/\bragcore\b/gu, "examplerag"],
+  [/\bPOKEVAULT\b/gu, "EXAMPLE_VISION"],
+  [/\bPokeVault\b/gu, "ExampleVision"],
+  [/\bpokevault\b/gu, "examplevision"],
+  [/\bNETOPS-FORGE\b/gu, "EXAMPLE-OPS"],
+  [/\bNetOps Forge\b/gu, "ExampleOps"],
+  [/\bnetops-forge\b/gu, "example-ops"],
+  [/\bTOWER\b/gu, "GPU_NODE"],
+  [/\bTower\b/gu, "GPU Node"],
+  [/\btower\b/gu, "gpu_node"]
+];
+
+const managedValidationWorkflow = ".gitea/workflows/managed-validation.yml";
+const pullRequestTrigger = /\n  pull_request:\r?\n/u;
+
+function publicPath(relativePath) {
+  let value = relativePath;
+  for (const [expression, replacement] of replacements) value = value.replace(expression, replacement);
+  return value;
+}
+
+function publicBytes(relativePath, sourceBytes) {
+  if (sourceBytes.includes(0)) return sourceBytes;
+  let value = sourceBytes.toString("utf8");
+  for (const [expression, replacement] of replacements) value = value.replace(expression, replacement);
+  if (relativePath === managedValidationWorkflow) {
+    if (!pullRequestTrigger.test(value)) {
+      fail(`Public export blocked: ${managedValidationWorkflow} has no expected pull_request trigger.`);
+    }
+    value = value.replace(
+      pullRequestTrigger,
+      "\n  # Public exports require explicit owner dispatch; fork PRs never reach private runners.\n"
+    );
+  }
+  return Buffer.from(value, "utf8");
+}
+
+const privatePatterns = [
+  { id: "private-ipv4", expression: /\b(?:10(?:\.\d{1,3}){3}|172\.(?:1[6-9]|2\d|3[01])(?:\.\d{1,3}){2}|192\.168(?:\.\d{1,3}){2})\b/u },
+  { id: "private-git-host", expression: /\bgitea\.itworx\.tech\b/iu },
+  { id: "personal-windows-path", expression: /[A-Z]:\\Users\\[^\\\s]+\\/iu },
+  { id: "private-ssh-command", expression: /\bssh\s+(?:root@|unraid\b)/iu },
+  { id: "private-node-name", expression: /\btower\b/iu },
+  { id: "private-integration-name", expression: /\b(?:examplerag|examplevision|example-ops)\b/iu }
+];
+
+const tracked = execFileSync("git", ["-C", root, "ls-files", "-z"], { encoding: "utf8" })
+  .split("\0")
+  .filter(Boolean)
+  .map((entry) => entry.replaceAll("\\", "/"));
+const files = tracked.filter(selected).sort((left, right) => left.localeCompare(right, "en"));
+if (files.length === 0) fail("The allowlist selected no tracked files.");
+
+const findings = [];
+const rendered = new Map();
+for (const relativePath of files) {
+  if (deniedPrefixes.some((prefix) => relativePath.startsWith(prefix))) {
+    findings.push({ rule: "denied-path", path: relativePath });
+    continue;
+  }
+  const source = path.join(root, ...relativePath.split("/"));
+  const stat = fs.lstatSync(source);
+  if (!stat.isFile() || stat.isSymbolicLink()) {
+    findings.push({ rule: "non-regular-file", path: relativePath });
+    continue;
+  }
+  if (stat.size > 10 * 1024 * 1024) {
+    findings.push({ rule: "oversized-file", path: relativePath, bytes: stat.size });
+    continue;
+  }
+  const bytes = publicBytes(relativePath, fs.readFileSync(source));
+  rendered.set(relativePath, bytes);
+  if (!bytes.includes(0)) {
+    const text = bytes.toString("utf8");
+    for (const rule of privatePatterns) {
+      if (rule.expression.test(text)) findings.push({ rule: rule.id, path: relativePath });
+    }
+  }
+}
+
+const destinations = new Map();
+for (const relativePath of files) {
+  const destinationPath = publicPath(relativePath);
+  const normalized = path.posix.normalize(destinationPath);
+  if (
+    destinationPath !== normalized ||
+    path.posix.isAbsolute(destinationPath) ||
+    destinationPath.startsWith("../") ||
+    destinationPath.includes("\\")
+  ) {
+    findings.push({ rule: "unsafe-destination-path", path: relativePath, destinationPath });
+    continue;
+  }
+  const collisionKey = destinationPath.toLocaleLowerCase("en-US");
+  const previous = destinations.get(collisionKey);
+  if (previous) {
+    findings.push({
+      rule: "destination-path-collision",
+      path: relativePath,
+      destinationPath,
+      conflictsWith: previous
+    });
+  } else {
+    destinations.set(collisionKey, relativePath);
+  }
+}
+
+for (const required of ["README.md", "SECURITY.md", "CONTRIBUTING.md", "LICENSE", "docker-compose.yml"]) {
+  if (!files.includes(required)) findings.push({ rule: "missing-required-file", path: required });
+}
+
+const sourceRevision = execFileSync("git", ["-C", root, "rev-parse", "HEAD"], { encoding: "utf8" }).trim();
+const report = { schemaVersion: 1, sourceRevision, selectedFiles: files.length, findings };
+if (reportPath) {
+  fs.mkdirSync(path.dirname(reportPath), { recursive: true });
+  fs.writeFileSync(reportPath, `${JSON.stringify(report, null, 2)}\n`, "utf8");
+}
+if (findings.length > 0) {
+  fail(`Public export blocked by ${findings.length} finding(s). See ${reportPath ?? "the scan output"}.`);
+}
+
+fs.mkdirSync(output, { recursive: false });
+const manifest = [];
+for (const relativePath of files) {
+  const bytes = rendered.get(relativePath);
+  const destinationPath = publicPath(relativePath);
+  const destination = path.join(output, ...destinationPath.split("/"));
+  fs.mkdirSync(path.dirname(destination), { recursive: true });
+  fs.writeFileSync(destination, bytes);
+  manifest.push({
+    path: destinationPath,
+    bytes: bytes.length,
+    sha256: crypto.createHash("sha256").update(bytes).digest("hex")
+  });
+}
+
+fs.writeFileSync(path.join(output, "PUBLIC_SOURCE_EXPORT.md"),
+  `# Curated public source export\n\nGenerated from private canonical revision \`${sourceRevision}\`.\n\n` +
+  "This parentless candidate excludes private operational history and uses synthetic example identifiers.\n",
+  "utf8");
+fs.writeFileSync(path.join(output, "PUBLIC_SOURCE_MANIFEST.json"),
+  `${JSON.stringify({ schemaVersion: 1, sourceRevision, files: manifest }, null, 2)}\n`, "utf8");
+console.log(`Exported ${files.length} reviewed files to ${output}`);

scripts/generate_configuration_docs.py

@@ -0,0 +1,274 @@
+"""Generate `.env.example` and `docs/CONFIGURATION.md` from the typed settings.
+
+Hand-maintained configuration documentation drifts, and the operator discovers the drift when a
+production deployment does something the manual said it would not. Both files are generated from
+`Settings` joined with `SETTING_DOCS`, and a test fails when the committed files no longer match —
+so adding a setting without documenting it breaks the build rather than shipping quietly.
+
+    python scripts/generate_configuration_docs.py          # write both files
+    python scripts/generate_configuration_docs.py --check  # fail if they are stale
+"""
+
+from __future__ import annotations
+
+import argparse
+import sys
+from pathlib import Path, PurePath
+from types import UnionType
+from typing import Any, Literal, Union, get_args, get_origin
+
+ROOT = Path(__file__).resolve().parents[1]
+sys.path.insert(0, str(ROOT / "backend" / "src"))
+
+from modelforge_api.domain.configuration_reference import (  # noqa: E402
+    DEPLOYMENT_DOCS,
+    SETTING_DOCS,
+    Sensitivity,
+)
+from modelforge_api.domain.release import (  # noqa: E402
+    MINIMUM_POSTGRES_MAJOR,
+    PRODUCT_NAME,
+    PRODUCT_VERSION,
+)
+from modelforge_api.settings import Settings  # noqa: E402
+
+ENV_PREFIX = "MODELFORGE_"
+
+#: Values a generated example must never contain. The example is committed to the repository, so a
+#: real secret placed here would be published with it.
+EXAMPLE_SECRET_PLACEHOLDER = ""
+
+#: Settings a container sets for itself. Listing them in the example invites an operator to override
+#: a path that only makes sense inside the image.
+CONTAINER_MANAGED = frozenset(
+    {
+        "config_root",
+        "alembic_directory",
+        "build_commit",
+        "build_timestamp",
+        "build_image_digest",
+        "agent_protocol_version",
+    }
+)
+
+
+def _default(name: str) -> str:
+    field = Settings.model_fields[name]
+    default: Any = field.default
+    if default is None:
+        return ""
+    if isinstance(default, bool):
+        return "true" if default else "false"
+    if isinstance(default, PurePath):
+        # These are paths *inside a Linux container*, so they must render with forward slashes
+        # whatever platform generates the file. `str(Path("/data/backups"))` gives `\dataackups`
+        # on Windows, and the committed .env.example shipped exactly that — telling operators to
+        # point a Linux container at a Windows path. It also made the generated files differ by
+        # platform, so the freshness check passed on one and failed on the other.
+        return default.as_posix()
+    if callable(default):  # pragma: no cover - default_factory fields
+        return ""
+    return str(default)
+
+
+def _render_annotation(annotation: Any) -> str:
+    """A stable, human-readable name for a setting's type.
+
+    Deliberately not `str(annotation)`: Python 3.13 moved Path into `pathlib._local`, so the same
+    field rendered as `pathlib.Path` on 3.12 and `pathlib._local.Path` on 3.13. The generated
+    reference then differed by interpreter version and the freshness check failed on whichever
+    machine had not produced it.
+    """
+
+    simple = {
+        "str": "string",
+        "int": "integer",
+        "float": "number",
+        "bool": "boolean",
+        "Path": "path",
+        "PosixPath": "path",
+        "WindowsPath": "path",
+        "SecretStr": "secret",
+        "NoneType": "none",
+    }
+    origin = get_origin(annotation)
+    if origin is Literal:
+        # Comma-separated, not pipe-separated: these land in a markdown table cell, and a pipe
+        # there silently splits the row into extra columns.
+        return ", ".join(f"`{value}`" for value in get_args(annotation))
+    if origin in (Union, UnionType):
+        parts = [
+            _render_annotation(argument)
+            for argument in get_args(annotation)
+            if argument is not type(None)
+        ]
+        rendered = " or ".join(dict.fromkeys(parts))
+        return f"{rendered}, optional"
+    if isinstance(annotation, type):
+        return simple.get(annotation.__name__, annotation.__name__)
+    return simple.get(str(annotation), str(annotation))
+
+
+def _type_name(name: str) -> str:
+    return _render_annotation(Settings.model_fields[name].annotation)
+
+
+def render_env_example() -> str:
+    lines = [
+        f"# {PRODUCT_NAME} {PRODUCT_VERSION} — configuration example",
+        "#",
+        "# Generated by scripts/generate_configuration_docs.py. Do not edit by hand.",
+        "# Copy to .env and fill in the values marked REQUIRED. See docs/CONFIGURATION.md.",
+        "#",
+        "# Secrets are intentionally empty here. This file is committed to the repository, so a",
+        "# real value placed in it would be published with the release.",
+        "",
+    ]
+    for name, doc in SETTING_DOCS.items():
+        if name in CONTAINER_MANAGED:
+            continue
+        marker = " (REQUIRED in production)" if doc.required_in_production else ""
+        secret = doc.sensitivity is Sensitivity.SECRET
+        lines.append(f"# {doc.description}{marker}")
+        value = EXAMPLE_SECRET_PLACEHOLDER if secret else _default(name)
+        lines.append(f"{ENV_PREFIX}{name.upper()}={value}")
+        lines.append("")
+
+    lines += [
+        "# --------------------------------------------------------------------------",
+        "# Deployment variables. Read by Compose, the Node Agent and the Runtime Worker",
+        "# rather than by the control-plane process - an operator still has to set them.",
+        "# --------------------------------------------------------------------------",
+        "",
+    ]
+    for name, doc in DEPLOYMENT_DOCS.items():
+        marker = " (REQUIRED in production)" if doc.required_in_production else ""
+        lines.append(f"# {doc.description}{marker}")
+        lines.append(f"{name}=")
+        lines.append("")
+    return "\n".join(lines).rstrip("\n") + "\n"
+
+
+def render_configuration_doc() -> str:
+    required = [name for name, doc in SETTING_DOCS.items() if doc.required_in_production]
+    secrets = [
+        name for name, doc in SETTING_DOCS.items() if doc.sensitivity is Sensitivity.SECRET
+    ]
+    lines = [
+        "# Configuration reference",
+        "",
+        f"Generated from the typed settings by `scripts/generate_configuration_docs.py` for "
+        f"{PRODUCT_NAME} {PRODUCT_VERSION}. Every setting the control plane reads appears here; a "
+        "setting added without documentation fails the build.",
+        "",
+        "All settings are environment variables with the `MODELFORGE_` prefix, read from the "
+        "process environment or from `.env`.",
+        "",
+        "## Required in production",
+        "",
+        "`MODELFORGE_ENV=production` turns on fail-closed startup validation. With it set, the "
+        "control plane refuses to start unless each of these is present and sound:",
+        "",
+    ]
+    for name in required:
+        lines.append(f"- `{ENV_PREFIX}{name.upper()}` — {SETTING_DOCS[name].description}")
+    lines += [
+        "",
+        "Production additionally refuses: a well-known development database password, a wildcard "
+        "CORS origin, remote model code execution, an unwritable storage root, an unsupported "
+        f"schema revision, a PostgreSQL major below {MINIMUM_POSTGRES_MAJOR}, and three policy "
+        "combinations that cannot all hold at once.",
+        "",
+        "## Generating secrets",
+        "",
+        "ModelForge never mints its own credentials — a platform that generates its own admin "
+        "secret has no way to tell you it did. Generate them yourself and store them outside the "
+        "deployment:",
+        "",
+        "```bash",
+        "# Operator API key (at least 32 characters)",
+        "python -c \"import secrets; print(secrets.token_urlsafe(48))\"",
+        "",
+        "# Backup encryption key (base64 AES-256)",
+        "python -c \"import base64, os; print(base64.b64encode(os.urandom(32)).decode())\"",
+        "",
+        "# Database password",
+        "python -c \"import secrets; print(secrets.token_urlsafe(32))\"",
+        "```",
+        "",
+        "Losing the backup encryption key makes every existing backup unrecoverable. It is the one "
+        "value that must be stored somewhere the deployment cannot take down with it.",
+        "",
+        "## Sensitivity",
+        "",
+        f"{len(secrets)} settings are credentials. They are never logged, never written to a "
+        "release artefact and never echoed in an error response:",
+        "",
+    ]
+    for name in secrets:
+        lines.append(f"- `{ENV_PREFIX}{name.upper()}`")
+    lines += ["", "## Every setting", "", "| Variable | Type | Default | Required | Sensitivity | Description |", "| --- | --- | --- | --- | --- | --- |"]
+    for name, doc in SETTING_DOCS.items():
+        default = _default(name)
+        rendered = f"`{default}`" if default else "—"
+        if doc.sensitivity is Sensitivity.SECRET:
+            rendered = "—"
+        lines.append(
+            f"| `{ENV_PREFIX}{name.upper()}` | {_type_name(name)} | {rendered} | "
+            f"{'yes' if doc.required_in_production else 'no'} | {doc.sensitivity} | "
+            f"{doc.description} |"
+        )
+    lines += [
+        "",
+        "## Deployment variables",
+        "",
+        "Read by Compose, the Node Agent and the Runtime Worker rather than by the control-plane "
+        "process. An operator still has to set them, so they are documented here too.",
+        "",
+        "| Variable | Required | Sensitivity | Description |",
+        "| --- | --- | --- | --- |",
+    ]
+    for name, doc in DEPLOYMENT_DOCS.items():
+        lines.append(
+            f"| `{name}` | {'yes' if doc.required_in_production else 'no'} | "
+            f"{doc.sensitivity} | {doc.description} |"
+        )
+    lines += [
+        "",
+        "## Restarts",
+        "",
+        "Every setting is read at process start. Changing any of them requires restarting the "
+        "control plane; none is re-read from the environment while the process is running.",
+        "",
+    ]
+    return "\n".join(lines)
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--check", action="store_true")
+    args = parser.parse_args(argv)
+
+    targets = {
+        ROOT / ".env.example": render_env_example(),
+        ROOT / "docs" / "CONFIGURATION.md": render_configuration_doc(),
+    }
+    stale: list[str] = []
+    for path, content in targets.items():
+        current = path.read_text("utf-8") if path.is_file() else None
+        if current == content:
+            print(f"  current  {path.relative_to(ROOT)}")
+            continue
+        stale.append(str(path.relative_to(ROOT)))
+        if not args.check:
+            path.parent.mkdir(parents=True, exist_ok=True)
+            path.write_text(content, encoding="utf-8")
+            print(f"  written  {path.relative_to(ROOT)}")
+    if args.check and stale:
+        print(f"stale: {', '.join(stale)} — run scripts/generate_configuration_docs.py")
+        return 1
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/m14_live_rehearsal.py

@@ -0,0 +1,313 @@
+"""Run bounded M14 alert lifecycle rehearsals against the configured database.
+
+The harness creates only isolated, recognisably named LAB fixtures. It never changes an
+existing node, deployment, migration, lifecycle operation, artifact, alias, or external
+process. Alert and incident history remains; transient queue/migration/rollback triggers
+are removed so their alerts can prove recovery.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import uuid
+from datetime import UTC, datetime, timedelta
+from typing import Any, TypeVar
+
+from modelforge_api.db import engine
+from modelforge_api.domain.observability import AlertAction
+from modelforge_api.persistence.models import (
+    Accelerator,
+    AlertHistoryEvent,
+    CapabilityDeployment,
+    ComputeNode,
+    LifecycleOperation,
+    MigrationPlan,
+    OperationalAlert,
+    SchedulerAcceleratorState,
+    ServingJob,
+    StorageRoot,
+)
+from modelforge_api.services.observability import ObservabilityService
+from sqlalchemy import delete, inspect, select
+from sqlalchemy.orm import Session
+
+ModelT = TypeVar("ModelT")
+CONFIRMATION = "--confirm-live-isolated-fixtures"
+
+
+def clone_row(model: type[ModelT], source: ModelT, **overrides: Any) -> ModelT:
+    """Clone mapped scalar columns while leaving identity/timestamps to the database."""
+    omitted = {"id", "created_at", "updated_at"}
+    values = {
+        column.key: getattr(source, column.key)
+        for column in inspect(model).columns
+        if column.key not in omitted
+    }
+    values.update(overrides)
+    return model(**values)
+
+
+def history(session: Session, alert_id: uuid.UUID) -> list[dict[str, Any]]:
+    rows = session.scalars(
+        select(AlertHistoryEvent)
+        .where(AlertHistoryEvent.alert_id == alert_id)
+        .order_by(AlertHistoryEvent.occurred_at)
+    )
+    return [
+        {
+            "state": row.to_state,
+            "at": row.occurred_at.isoformat(),
+            "actor": row.actor,
+        }
+        for row in rows
+    ]
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser()
+    parser.add_argument(CONFIRMATION, action="store_true")
+    args = parser.parse_args()
+    if not args.confirm_live_isolated_fixtures:
+        parser.error(f"explicit {CONFIRMATION} is required")
+
+    run_id = uuid.uuid4().hex[:12]
+    now = datetime.now(UTC)
+    first_observation = now - timedelta(seconds=90)
+    firing_observation = now - timedelta(seconds=20)
+
+    with Session(engine) as session:
+        service = ObservabilityService(session, actor="m14-live-rehearsal")
+        service.ensure_defaults()
+
+        source_deployment = session.scalar(
+            select(CapabilityDeployment)
+            .where(CapabilityDeployment.production.is_(False))
+            .order_by(CapabilityDeployment.created_at)
+            .limit(1)
+        )
+        source_migration = session.scalar(
+            select(MigrationPlan).order_by(MigrationPlan.created_at).limit(1)
+        )
+        source_lifecycle = session.scalar(
+            select(LifecycleOperation).order_by(LifecycleOperation.started_at).limit(1)
+        )
+        if not source_deployment or not source_migration or not source_lifecycle:
+            raise RuntimeError(
+                "required pre-existing LAB rehearsal sources are unavailable"
+            )
+
+        node = ComputeNode(
+            key=f"m14-rehearsal-node-{run_id}",
+            hostname=f"m14-rehearsal-{run_id}.invalid",
+            display_name=f"M14 isolated rehearsal {run_id}",
+            identity_source="m14_rehearsal",
+            status="active",
+            inventory={"fixture": "M14"},
+            enabled=True,
+            production_eligible=True,
+            lab_eligible=True,
+            benchmark_eligible=False,
+            observation_source="local_control_plane",
+            liveness_state="offline",
+            last_heartbeat_at=first_observation,
+            total_ram_bytes=8 * 1024**3,
+            labels={"purpose": "m14_live_rehearsal", "run_id": run_id},
+        )
+        session.add(node)
+        session.flush()
+
+        accelerator = Accelerator(
+            compute_node_id=node.id,
+            device_index=0,
+            device_uuid=f"M14-REHEARSAL-{run_id}",
+            name="M14 isolated virtual GPU fixture",
+            vendor="NVIDIA",
+            total_vram_bytes=4 * 1024**3,
+            memory_total_mb=4096,
+            status="active",
+            inventory_source="m14_rehearsal",
+            capabilities={"fixture": True},
+        )
+        session.add(accelerator)
+        session.flush()
+        gpu_state = SchedulerAcceleratorState(
+            accelerator_id=accelerator.id,
+            pressure_state="HIGH",
+            pressure_changed_at=first_observation,
+            last_observed_at=first_observation,
+        )
+        session.add(gpu_state)
+
+        storage = StorageRoot(
+            compute_node_id=node.id,
+            name=f"m14-rehearsal-storage-{run_id}",
+            purpose="m14_rehearsal",
+            path=f"fixture://m14/{run_id}",
+            status="ready",
+            writable=False,
+            capacity_bytes=100 * 1024**3,
+            free_bytes=12 * 1024**3,
+            reserve_bytes=10 * 1024**3,
+            reserve_percent=10,
+            capacity_observed_at=first_observation,
+            validation_details={"fixture": True, "no_real_storage": True},
+        )
+        session.add(storage)
+
+        lab_deployment = clone_row(
+            CapabilityDeployment,
+            source_deployment,
+            compute_node_id=node.id,
+            accelerator_id=accelerator.id,
+            channel="m14_rehearsal",
+            status="stable",
+            production=False,
+            health_status="unavailable",
+            routing_weight=0,
+            config_fingerprint=(f"m14{run_id}" * 6)[:64],
+            provenance={"fixture": "M14", "run_id": run_id, "routable": False},
+            promoted_at=None,
+            draining_at=None,
+            deprecated_at=None,
+        )
+        session.add(lab_deployment)
+        session.flush()
+        lab_deployment_id = lab_deployment.id
+
+        queue_jobs = [
+            ServingJob(
+                capability_deployment_id=lab_deployment.id,
+                compute_node_id=node.id,
+                operation="invoke",
+                status="queued",
+                priority="background",
+                idempotency_key=f"m14-{run_id}-queue-{index}",
+                payload_reference=None,
+                result_summary={"fixture": True},
+            )
+            for index in range(9)
+        ]
+        session.add_all(queue_jobs)
+
+        migration = clone_row(
+            MigrationPlan,
+            source_migration,
+            environment="LAB",
+            target_shadow_target=f"m14-rehearsal-shadow-{run_id}",
+            idempotency_key=f"m14-rehearsal-{run_id}",
+            state="FAILED",
+            plan_fingerprint=(f"a{run_id}" * 6)[:64],
+            approval_fingerprint=(f"b{run_id}" * 6)[:64],
+            failure_code="VALIDATION_FAILED",
+            failure_details={
+                "fixture": True,
+                "reason": "controlled M14 validation failure",
+            },
+            started_at=first_observation,
+            completed_at=first_observation,
+        )
+        session.add(migration)
+
+        rollback = clone_row(
+            LifecycleOperation,
+            source_lifecycle,
+            stage="ROLLED_BACK",
+            idempotency_key=f"m14-rehearsal-rollback-{run_id}",
+            failure_code="ROLLBACK_FAILED",
+            failure_details={"fixture": True, "reason": "controlled adapter refusal"},
+            started_at=first_observation,
+            finished_at=first_observation,
+        )
+        session.add(rollback)
+        session.commit()
+
+        service.evaluate_alerts(first_observation)
+        service.evaluate_alerts(firing_observation)
+
+        expected_subjects = {
+            "NODE_OFFLINE": str(node.id),
+            "GPU_PRESSURE": str(accelerator.id),
+            "STORAGE_LOW": str(storage.id),
+            "CAPABILITY_UNAVAILABLE": None,
+            "QUEUE_SATURATION": "scheduler-queue",
+            "MIGRATION_FAILURE": str(migration.id),
+            "ROLLBACK_FAILURE": str(rollback.id),
+        }
+        rehearsed: dict[str, OperationalAlert] = {}
+        for alert_type, subject_ref in expected_subjects.items():
+            statement = select(OperationalAlert).where(
+                OperationalAlert.alert_type == alert_type
+            )
+            if subject_ref is not None:
+                statement = statement.where(OperationalAlert.subject_ref == subject_ref)
+            else:
+                statement = statement.where(
+                    OperationalAlert.source == "capability_deployment",
+                    OperationalAlert.details["node_id"].as_string() == str(node.id),
+                )
+            alert = session.scalar(
+                statement.order_by(OperationalAlert.first_seen_at.desc())
+            )
+            if not alert or alert.state != "FIRING":
+                raise RuntimeError(f"{alert_type} did not reach FIRING")
+            rehearsed[alert_type] = alert
+
+        node_ack = service.acknowledge(
+            rehearsed["NODE_OFFLINE"].id,
+            AlertAction(
+                actor="m14-rehearsal-operator",
+                reason="Controlled isolated node rehearsal acknowledged",
+            ),
+        )
+        if node_ack.state.value != "ACKNOWLEDGED":
+            raise RuntimeError("node acknowledgement was not persisted")
+
+        node.liveness_state = "online"
+        node.last_heartbeat_at = now
+        node.enabled = False
+        node.status = "unavailable"
+        accelerator.status = "missing"
+        gpu_state.pressure_state = "NORMAL"
+        gpu_state.last_observed_at = now
+        storage.free_bytes = 90 * 1024**3
+        storage.deprecated_at = now
+        lab_deployment.health_status = "ready_on_demand"
+        session.execute(
+            delete(ServingJob).where(ServingJob.id.in_([row.id for row in queue_jobs]))
+        )
+        session.delete(lab_deployment)
+        session.delete(migration)
+        session.delete(rollback)
+        session.commit()
+
+        service.evaluate_alerts(now + timedelta(seconds=1))
+        session.expire_all()
+
+        result: dict[str, Any] = {
+            "run_id": run_id,
+            "fixture_node_id": str(node.id),
+            "fixture_deployment_id": str(lab_deployment_id),
+            "queue_depth_trigger": len(queue_jobs),
+            "first_observation": first_observation.isoformat(),
+            "firing_observation": firing_observation.isoformat(),
+            "recovery_observation": (now + timedelta(seconds=1)).isoformat(),
+            "alerts": {},
+        }
+        for alert_type, prior in rehearsed.items():
+            current = session.get(OperationalAlert, prior.id)
+            if not current or current.state != "RESOLVED":
+                raise RuntimeError(f"{alert_type} did not reach RESOLVED")
+            result["alerts"][alert_type] = {
+                "id": str(current.id),
+                "severity": current.severity,
+                "state": current.state,
+                "occurrences": current.occurrence_count,
+                "history": history(session, current.id),
+            }
+
+        print(json.dumps(result, indent=2, sort_keys=True))
+
+
+if __name__ == "__main__":
+    main()

scripts/m14_live_soak.ps1

@@ -0,0 +1,218 @@
+param(
+    [int]$Cycles = 4,
+    [int]$IdleSeconds = 45
+)
+
+$ErrorActionPreference = "Stop"
+if ($Cycles -lt 2 -or $Cycles -gt 12) {
+    throw "Cycles must be between 2 and 12"
+}
+if ($IdleSeconds -lt 10 -or $IdleSeconds -gt 300) {
+    throw "IdleSeconds must be between 10 and 300"
+}
+
+$baseUrl = "http://127.0.0.1:8000"
+$operatorLine = Get-Content .env | Where-Object {
+    $_ -match '^MODELFORGE_OPERATOR_API_KEY='
+} | Select-Object -First 1
+if (-not $operatorLine) {
+    throw "MODELFORGE_OPERATOR_API_KEY is unavailable"
+}
+$operatorToken = $operatorLine.Substring($operatorLine.IndexOf('=') + 1).Trim().Trim('"').Trim("'")
+$operatorHeaders = @{ "X-ModelForge-Admin-Token" = $operatorToken }
+$runId = [guid]::NewGuid().ToString("N").Substring(0, 12)
+$clientId = $null
+$results = [System.Collections.Generic.List[object]]::new()
+$startedAt = [datetime]::UtcNow
+
+function New-SilenceWavBase64 {
+    $sampleRate = 16000
+    $sampleCount = $sampleRate
+    $dataLength = $sampleCount * 2
+    $stream = [System.IO.MemoryStream]::new()
+    $writer = [System.IO.BinaryWriter]::new($stream)
+    try {
+        $writer.Write([System.Text.Encoding]::ASCII.GetBytes("RIFF"))
+        $writer.Write([int](36 + $dataLength))
+        $writer.Write([System.Text.Encoding]::ASCII.GetBytes("WAVE"))
+        $writer.Write([System.Text.Encoding]::ASCII.GetBytes("fmt "))
+        $writer.Write([int]16)
+        $writer.Write([int16]1)
+        $writer.Write([int16]1)
+        $writer.Write([int]$sampleRate)
+        $writer.Write([int]($sampleRate * 2))
+        $writer.Write([int16]2)
+        $writer.Write([int16]16)
+        $writer.Write([System.Text.Encoding]::ASCII.GetBytes("data"))
+        $writer.Write([int]$dataLength)
+        $writer.Write([byte[]]::new($dataLength))
+        $writer.Flush()
+        return [Convert]::ToBase64String($stream.ToArray())
+    }
+    finally {
+        $writer.Dispose()
+        $stream.Dispose()
+    }
+}
+
+function Invoke-SoakCapability {
+    param(
+        [string]$Capability,
+        [string]$Path,
+        [hashtable]$Body,
+        [int]$Cycle,
+        [hashtable]$Headers
+    )
+    $watch = [System.Diagnostics.Stopwatch]::StartNew()
+    try {
+        $response = Invoke-RestMethod -Method Post -Uri "$baseUrl$Path" -Headers $Headers `
+            -ContentType "application/json" -Body ($Body | ConvertTo-Json -Depth 8 -Compress) `
+            -TimeoutSec 180
+        $watch.Stop()
+        $results.Add([pscustomobject]@{
+            cycle = $Cycle
+            capability = $Capability
+            success = $true
+            request_id = [string]$response.request_id
+            cold = [bool]$response.execution.cold
+            residency = [string]$response.execution.residency
+            node = [string]$response.execution.node
+            queue_ms = [double]$response.execution.timings.queue_ms
+            load_ms = [double]$response.execution.timings.load_ms
+            inference_ms = [double]$response.execution.timings.inference_ms
+            total_ms = [double]$response.execution.timings.total_ms
+            wall_ms = [double]$watch.Elapsed.TotalMilliseconds
+            error_class = $null
+            error_code = $null
+        })
+    }
+    catch {
+        $watch.Stop()
+        $errorCode = $null
+        if ($_.ErrorDetails.Message) {
+            try {
+                $errorBody = $_.ErrorDetails.Message | ConvertFrom-Json
+                $errorCode = [string]$errorBody.error.code
+            }
+            catch {
+                $errorCode = "UNPARSEABLE_HTTP_ERROR"
+            }
+        }
+        $results.Add([pscustomobject]@{
+            cycle = $Cycle
+            capability = $Capability
+            success = $false
+            request_id = $null
+            cold = $null
+            residency = $null
+            node = $null
+            queue_ms = $null
+            load_ms = $null
+            inference_ms = $null
+            total_ms = $null
+            wall_ms = [double]$watch.Elapsed.TotalMilliseconds
+            error_class = $_.Exception.GetType().Name
+            error_code = $errorCode
+        })
+    }
+}
+
+function Get-Percentile {
+    param([double[]]$Values, [double]$Percentile)
+    if (-not $Values -or $Values.Count -eq 0) {
+        return $null
+    }
+    $ordered = @($Values | Sort-Object)
+    $index = [math]::Ceiling($Percentile * $ordered.Count) - 1
+    return [double]$ordered[[math]::Max(0, $index)]
+}
+
+try {
+    $clientBody = @{
+        name = "m14-soak-$runId"
+        allowed_capabilities = @(
+            "rag.embedding@1",
+            "vision.embedding@1",
+            "speech.transcription@1"
+        )
+        requests_per_minute = 60
+        max_concurrent_requests = 1
+        workload_priority = "interactive"
+        credential_expires_at = [datetime]::UtcNow.AddMinutes(30).ToString("o")
+    }
+    $client = Invoke-RestMethod -Method Post -Uri "$baseUrl/api/v1/admin/service-clients" `
+        -Headers $operatorHeaders -ContentType "application/json" `
+        -Body ($clientBody | ConvertTo-Json -Depth 6 -Compress)
+    $clientId = [string]$client.id
+    $serviceHeaders = @{ Authorization = "Bearer $($client.credential)" }
+    $audio = New-SilenceWavBase64
+
+    for ($cycle = 1; $cycle -le $Cycles; $cycle++) {
+        Invoke-RestMethod -Method Post -Uri "$baseUrl/api/v1/admin/operations/capacity/collect" `
+            -Headers $operatorHeaders | Out-Null
+
+        Invoke-SoakCapability -Capability "rag.embedding@1" `
+            -Path "/api/v1/capabilities/rag.embedding@1/invoke" `
+            -Body @{ input = @("M14 bounded operational soak cycle $cycle") } `
+            -Cycle $cycle -Headers $serviceHeaders
+        Invoke-SoakCapability -Capability "vision.embedding@1 LAB" `
+            -Path "/api/v1/capabilities/vision.embedding@1/invoke" `
+            -Body @{ items = @(@{ text = "M14 visual control sample cycle $cycle" }) } `
+            -Cycle $cycle -Headers $serviceHeaders
+        Invoke-SoakCapability -Capability "speech.transcription@1 LAB" `
+            -Path "/api/v1/capabilities/speech.transcription@1/invoke" `
+            -Body @{ audio_base64 = $audio; media_type = "audio/wav"; language = "en" } `
+            -Cycle $cycle -Headers $serviceHeaders
+
+        Invoke-RestMethod -Method Post -Uri "$baseUrl/api/v1/admin/operations/capacity/collect" `
+            -Headers $operatorHeaders | Out-Null
+        Invoke-RestMethod -Method Post -Uri "$baseUrl/api/v1/admin/operations/slo-evaluations/run" `
+            -Headers $operatorHeaders | Out-Null
+        Invoke-RestMethod -Method Post -Uri "$baseUrl/api/v1/admin/operations/alerts/evaluate" `
+            -Headers $operatorHeaders | Out-Null
+
+        if ($cycle -lt $Cycles) {
+            Start-Sleep -Seconds $IdleSeconds
+        }
+    }
+}
+finally {
+    if ($clientId) {
+        Invoke-RestMethod -Method Delete `
+            -Uri "$baseUrl/api/v1/admin/service-clients/$clientId/credential" `
+            -Headers $operatorHeaders | Out-Null
+    }
+}
+
+$finishedAt = [datetime]::UtcNow
+$summary = foreach ($group in ($results | Group-Object capability)) {
+    $successful = @($group.Group | Where-Object success)
+    $latencies = @($successful | ForEach-Object { [double]$_.total_ms })
+    [pscustomobject]@{
+        capability = $group.Name
+        requests = $group.Count
+        failures = @($group.Group | Where-Object { -not $_.success }).Count
+        cold = @($successful | Where-Object cold).Count
+        warm = @($successful | Where-Object { -not $_.cold }).Count
+        p50_ms = Get-Percentile -Values $latencies -Percentile 0.50
+        p95_ms = Get-Percentile -Values $latencies -Percentile 0.95
+        max_queue_ms = if ($successful) {
+            [double](($successful | Measure-Object queue_ms -Maximum).Maximum)
+        } else { $null }
+        max_load_ms = if ($successful) {
+            [double](($successful | Measure-Object load_ms -Maximum).Maximum)
+        } else { $null }
+    }
+}
+
+[pscustomobject]@{
+    run_id = $runId
+    started_at = $startedAt.ToString("o")
+    finished_at = $finishedAt.ToString("o")
+    duration_seconds = ($finishedAt - $startedAt).TotalSeconds
+    cycles = $Cycles
+    idle_seconds = $IdleSeconds
+    credential_revoked = [bool]$clientId
+    summary = @($summary)
+    requests = @($results)
+} | ConvertTo-Json -Depth 8

scripts/m15_background_writes.sh

@@ -0,0 +1,24 @@
+#!/usr/bin/env sh
+# Bounded background control-plane writes for the M15 backup-consistency rehearsal.
+#
+# A logical backup must stay coherent while ModelForge keeps writing. This drives real
+# operational writes (capacity snapshots and SLO evaluations) for a bounded period so the
+# dump is taken against a moving database rather than a quiesced one.
+set -eu
+
+BASE_URL="${MODELFORGE_BASE_URL:-http://localhost:8000}"
+DURATION="${M15_WRITE_SECONDS:-90}"
+INTERVAL="${M15_WRITE_INTERVAL:-3}"
+TOKEN="${MODELFORGE_OPERATOR_API_KEY:?operator API key is required}"
+
+deadline=$(( $(date +%s) + DURATION ))
+writes=0
+while [ "$(date +%s)" -lt "$deadline" ]; do
+  curl -s -o /dev/null -X POST -H "X-ModelForge-Admin-Token: ${TOKEN}" \
+    "${BASE_URL}/api/v1/admin/operations/capacity/collect" || true
+  curl -s -o /dev/null -X POST -H "X-ModelForge-Admin-Token: ${TOKEN}" \
+    "${BASE_URL}/api/v1/admin/operations/slo-evaluations/run" || true
+  writes=$(( writes + 2 ))
+  sleep "${INTERVAL}"
+done
+printf 'background_write_requests=%s\n' "${writes}"

scripts/m15_record_artifact_recovery.py

@@ -0,0 +1,116 @@
+"""Record observed artifact-recovery evidence against a planned recovery operation.
+
+The rehydration itself is performed by the existing acquisition plane: an exact-revision download
+plan is executed by the Node Agent, which quarantines, hashes and only then promotes. This script
+reads the *observed* result out of the control-plane database and journals it on the M15 artifact
+recovery operation, so the recovery record carries measured evidence rather than an assumption.
+
+    python scripts/m15_record_artifact_recovery.py \
+        --operation-id <uuid> --storage-root-id <uuid> --artifact-job-id <uuid> \
+        --download-plan-id <uuid> --database-url postgresql+psycopg://...
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from pathlib import Path
+
+from sqlalchemy import create_engine, select
+from sqlalchemy.orm import Session
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "backend" / "src"))
+
+from modelforge_api.domain.recovery import (  # noqa: E402
+    ArtifactRecoveryState,
+    RecoveryFailureCode,
+)
+from modelforge_api.persistence.models import (  # noqa: E402
+    ArtifactJob,
+    ArtifactLocation,
+    ModelArtifact,
+)
+from modelforge_api.services.recovery import RecoveryService  # noqa: E402
+from modelforge_api.settings import Settings  # noqa: E402
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--operation-id", required=True)
+    parser.add_argument("--storage-root-id", required=True)
+    parser.add_argument("--artifact-job-id", default=None)
+    parser.add_argument("--download-plan-id", default=None)
+    parser.add_argument("--database-url", required=True)
+    parser.add_argument(
+        "--blocked-reason",
+        default=None,
+        help="record an upstream outage as ARTIFACT_REHYDRATION_BLOCKED instead of a recovery",
+    )
+    args = parser.parse_args()
+
+    engine = create_engine(args.database_url, pool_pre_ping=True)
+    try:
+        with Session(engine) as session:
+            service = RecoveryService(
+                session, Settings(database_url=args.database_url), "operator", "m15-rehearsal"
+            )
+            if args.blocked_reason:
+                response = service.record_artifact_recovery(
+                    __import__("uuid").UUID(args.operation_id),
+                    state=ArtifactRecoveryState.BLOCKED,
+                    verified_files=[],
+                    bytes_recovered=0,
+                    duration_seconds=None,
+                    failure_code=RecoveryFailureCode.ARTIFACT_REHYDRATION_BLOCKED.value,
+                    failure_reason=args.blocked_reason,
+                )
+                print(json.dumps(json.loads(response.model_dump_json()), indent=2))
+                return 0
+
+            rows = session.execute(
+                select(ModelArtifact, ArtifactLocation)
+                .join(ArtifactLocation, ArtifactLocation.artifact_id == ModelArtifact.id)
+                .where(ArtifactLocation.storage_root_id == args.storage_root_id)
+                .order_by(ModelArtifact.filename)
+            ).all()
+            verified = [
+                {
+                    "filename": artifact.filename,
+                    "sha256": location.observed_sha256,
+                    "expected_sha256": artifact.sha256,
+                    "size_bytes": location.size_bytes,
+                    "relative_path": location.relative_path,
+                    "status": location.status,
+                    "security_status": artifact.security_status,
+                }
+                for artifact, location in rows
+            ]
+            duration = None
+            if args.artifact_job_id:
+                job = session.get(ArtifactJob, __import__("uuid").UUID(args.artifact_job_id))
+                if job and job.completed_at and job.started_at:
+                    duration = (job.completed_at - job.started_at).total_seconds()
+            response = service.record_artifact_recovery(
+                __import__("uuid").UUID(args.operation_id),
+                state=ArtifactRecoveryState.RECOVERED,
+                verified_files=verified,
+                bytes_recovered=sum(int(item["size_bytes"] or 0) for item in verified),
+                duration_seconds=duration,
+                download_plan_id=(
+                    __import__("uuid").UUID(args.download_plan_id)
+                    if args.download_plan_id
+                    else None
+                ),
+                artifact_job_id=(
+                    __import__("uuid").UUID(args.artifact_job_id) if args.artifact_job_id else None
+                ),
+            )
+            print(json.dumps(json.loads(response.model_dump_json()), indent=2))
+            return 0 if response.state is ArtifactRecoveryState.RECOVERED else 1
+    finally:
+        engine.dispose()
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/m16_chaos.py

@@ -0,0 +1,858 @@
+"""M16 bounded chaos harness.
+
+One harness, one shape, every scenario:
+
+    preflight -> baseline -> inject -> observe -> assert -> recover -> verify invariants -> cleanup
+
+Faults are injected from *outside* the product, through the container runtime and through
+seams that already exist for rehearsals (the M12 lifecycle pause, the M15 restore interruption).
+ModelForge itself gains no chaos endpoint and no way to run a shell: a control plane that can be
+told to break itself is a control plane an attacker can tell to break itself.
+
+Every scenario is bounded in time, reverses its own fault, and re-runs the full invariant set
+afterwards. A scenario that cannot restore the platform reports RECOVERY_FAILED rather than
+leaving the environment dirty and calling itself done.
+
+    python scripts/m16_chaos.py --list
+    python scripts/m16_chaos.py --scenario redis-outage --seed 1234
+    python scripts/m16_chaos.py --all --report out.json
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import random
+import subprocess  # noqa: S404 - fixed argv container-runtime control, never a shell
+import sys
+import time
+import urllib.error
+import urllib.request
+import uuid
+from collections.abc import Callable
+from dataclasses import dataclass, field
+from datetime import UTC, datetime
+from pathlib import Path
+from typing import Any
+
+from sqlalchemy import create_engine, text
+from sqlalchemy.orm import Session
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "backend" / "src"))
+
+from modelforge_api.services.invariants import (  # noqa: E402
+    InvariantStatus,
+    check_invariants,
+)
+
+DEFAULT_BASE_URL = "http://127.0.0.1:8000"
+DEFAULT_DATABASE_URL = os.getenv("MODELFORGE_RUNTIME_DATABASE_URL")
+
+API = "modelforge-api-1"
+REDIS = "modelforge-redis-1"
+POSTGRES = "modelforge-postgres-1"
+NODE_AGENT = "modelforge-node-agent-1"
+NETWORK = "modelforge_default"
+
+
+class ChaosError(RuntimeError):
+    """A harness failure: the fault could not be injected, or the platform could not be restored."""
+
+
+# --------------------------------------------------------------------- runtime control
+
+
+def docker(*args: str, check: bool = True, timeout: int = 120) -> str:
+    completed = subprocess.run(  # noqa: S603 - fixed argv, no shell
+        ["docker", *args],  # noqa: S607 - docker resolves from PATH
+        capture_output=True,
+        text=True,
+        encoding="utf-8",
+        errors="replace",
+        timeout=timeout,
+        check=False,
+    )
+    if check and completed.returncode != 0:
+        raise ChaosError(f"docker {' '.join(args)} failed: {completed.stderr.strip()[:400]}")
+    return (completed.stdout or "").strip()
+
+
+def container_running(name: str) -> bool:
+    return docker("inspect", "--format", "{{.State.Running}}", name, check=False) == "true"
+
+
+def container_started_at(name: str) -> str:
+    return docker("inspect", "--format", "{{.State.StartedAt}}", name, check=False)
+
+
+# --------------------------------------------------------------------- probes
+
+
+@dataclass
+class Probe:
+    ok: bool
+    status: int
+    latency_ms: float
+    body: str = ""
+
+
+def http(
+    url: str, *, token: str | None = None, method: str = "GET", timeout: float = 5.0
+) -> Probe:
+    request = urllib.request.Request(url, method=method)  # noqa: S310 - fixed http scheme
+    if token:
+        request.add_header("X-ModelForge-Admin-Token", token)
+    started = time.perf_counter()
+    try:
+        with urllib.request.urlopen(request, timeout=timeout) as response:  # noqa: S310
+            body = response.read(4096).decode("utf-8", "replace")
+            return Probe(True, response.status, (time.perf_counter() - started) * 1000, body)
+    except urllib.error.HTTPError as error:
+        return Probe(False, error.code, (time.perf_counter() - started) * 1000, "")
+    except (urllib.error.URLError, TimeoutError, OSError):
+        return Probe(False, 0, (time.perf_counter() - started) * 1000, "")
+
+
+def wait_for(predicate: Callable[[], bool], *, timeout: float, interval: float = 1.0) -> bool:
+    deadline = time.monotonic() + timeout
+    while time.monotonic() < deadline:
+        if predicate():
+            return True
+        time.sleep(interval)
+    return predicate()
+
+
+# --------------------------------------------------------------------- scenario contract
+
+
+@dataclass
+class ScenarioContext:
+    base_url: str
+    database_url: str
+    token: str
+    seed: int
+    rng: random.Random
+    observations: dict[str, Any] = field(default_factory=dict)
+
+    def note(self, key: str, value: Any) -> None:
+        self.observations[key] = value
+
+    def session(self) -> Session:
+        engine = create_engine(self.database_url, pool_pre_ping=True)
+        return Session(engine)
+
+
+@dataclass
+class Scenario:
+    key: str
+    title: str
+    failure_class: str
+    subsystem: str
+    expected: str
+    run: Callable[[ScenarioContext], None]
+    recover: Callable[[ScenarioContext], None] | None = None
+    cleanup: Callable[[ScenarioContext], None] | None = None
+    requires_docker: bool = True
+
+
+SCENARIOS: dict[str, Scenario] = {}
+
+
+def scenario(
+    key: str,
+    title: str,
+    failure_class: str,
+    subsystem: str,
+    expected: str,
+    *,
+    recover: Callable[[ScenarioContext], None] | None = None,
+    cleanup: Callable[[ScenarioContext], None] | None = None,
+    requires_docker: bool = True,
+) -> Callable[[Callable[[ScenarioContext], None]], Callable[[ScenarioContext], None]]:
+    def decorate(function: Callable[[ScenarioContext], None]) -> Callable[[ScenarioContext], None]:
+        SCENARIOS[key] = Scenario(
+            key=key,
+            title=title,
+            failure_class=failure_class,
+            subsystem=subsystem,
+            expected=expected,
+            run=function,
+            recover=recover,
+            cleanup=cleanup,
+            requires_docker=requires_docker,
+        )
+        return function
+
+    return decorate
+
+
+# --------------------------------------------------------------------- scenarios
+
+
+def _api_healthy(context: ScenarioContext) -> bool:
+    return http(f"{context.base_url}/api/v1/health/live").ok
+
+
+def _restore_api(context: ScenarioContext) -> None:
+    if not container_running(API):
+        docker("start", API)
+    if not wait_for(lambda: _api_healthy(context), timeout=120):
+        raise ChaosError("the control plane did not become healthy again")
+
+
+@scenario(
+    "redis-outage",
+    "Redis becomes unreachable and returns",
+    failure_class="DEPENDENCY",
+    subsystem="Redis",
+    expected="the control plane stays up, reports the dependency honestly, and replays nothing",
+    recover=_restore_api,
+)
+def redis_outage(context: ScenarioContext) -> None:
+    """Redis holds queues and transient payloads only; losing it must not corrupt authoritative state."""
+
+    before = http(f"{context.base_url}/api/v1/health/live")
+    context.note("api_healthy_before", before.ok)
+
+    docker("stop", REDIS)
+    context.note("redis_stopped", True)
+    time.sleep(5)
+
+    during_live = http(f"{context.base_url}/api/v1/health/live")
+    during_ready = http(f"{context.base_url}/api/v1/health/ready")
+    during_models = http(f"{context.base_url}/api/v1/models")
+    context.note("during_live_status", during_live.status)
+    context.note("during_ready_status", during_ready.status)
+    context.note("during_registry_status", during_models.status)
+
+    with context.session() as session:
+        queued = session.execute(
+            text("select count(*) from serving_jobs where status in ('queued','leased','running')")
+        ).scalar_one()
+        context.note("serving_jobs_in_flight_during_outage", int(queued))
+
+    docker("start", REDIS)
+    if not wait_for(lambda: container_running(REDIS), timeout=60):
+        raise ChaosError("redis did not restart")
+    time.sleep(5)
+    after = http(f"{context.base_url}/api/v1/health/live")
+    context.note("api_healthy_after", after.ok)
+
+    with context.session() as session:
+        duplicates = session.execute(
+            text(
+                "select count(*) from (select idempotency_key from serving_jobs "
+                "group by idempotency_key having count(*) > 1) s"
+            )
+        ).scalar_one()
+        context.note("duplicate_serving_idempotency_keys", int(duplicates))
+    if int(duplicates) != 0:
+        raise ChaosError("a Redis outage produced duplicate serving jobs")
+    if not after.ok:
+        raise ChaosError("the control plane did not recover after Redis returned")
+
+
+@scenario(
+    "postgres-restart",
+    "PostgreSQL restarts under the running control plane",
+    failure_class="DEPENDENCY",
+    subsystem="PostgreSQL",
+    expected="the API fails safely while the database is gone and recovers its pool without duplicate work",
+    recover=_restore_api,
+)
+def postgres_restart(context: ScenarioContext) -> None:
+    """The control plane may not invent state while its authoritative store is unavailable."""
+
+    with context.session() as session:
+        audit_before = session.execute(text("select count(*) from audit_events")).scalar_one()
+    context.note("audit_events_before", int(audit_before))
+
+    docker("stop", POSTGRES)
+    time.sleep(3)
+    during_live = http(f"{context.base_url}/api/v1/health/live")
+    during_models = http(f"{context.base_url}/api/v1/models", timeout=10.0)
+    context.note("during_live_status", during_live.status)
+    context.note("during_registry_status", during_models.status)
+    if during_models.ok:
+        raise ChaosError("the registry served data while PostgreSQL was stopped")
+
+    docker("start", POSTGRES)
+    if not wait_for(
+        lambda: docker(
+            "exec", POSTGRES, "pg_isready", "-U", "modelforge", "-d", "modelforge", check=False
+        ).endswith("accepting connections"),
+        timeout=120,
+    ):
+        raise ChaosError("PostgreSQL did not accept connections again")
+
+    recovered = wait_for(lambda: http(f"{context.base_url}/api/v1/models", timeout=10.0).ok, timeout=120)
+    context.note("registry_recovered_without_restart", recovered)
+
+    with context.session() as session:
+        audit_after = session.execute(text("select count(*) from audit_events")).scalar_one()
+        duplicates = session.execute(
+            text(
+                "select count(*) from (select sequence from audit_events "
+                "group by sequence having count(*) > 1) s"
+            )
+        ).scalar_one()
+    context.note("audit_events_after", int(audit_after))
+    context.note("duplicate_audit_sequences", int(duplicates))
+    if int(duplicates) != 0:
+        raise ChaosError("a PostgreSQL restart produced duplicate audit sequences")
+    if not recovered:
+        raise ChaosError("the control plane never recovered its database pool")
+
+
+RECOVERY_AGENT = "modelforge-node-recovery-agent-1"
+RECOVERY_COMPOSE = "docker-compose.node-recovery.yml"
+
+
+def _recovery_agent_node(context: ScenarioContext) -> tuple[Any, Any] | None:
+    with context.session() as session:
+        row = session.execute(
+            text(
+                "select id, last_heartbeat_at from compute_nodes "
+                "where hostname = 'modelforge-chaos-rehearsal' and enabled "
+                "order by first_seen_at desc limit 1"
+            )
+        ).first()
+    return (row[0], row[1]) if row else None
+
+
+def _stop_recovery_agent(context: ScenarioContext) -> None:
+    docker("network", "connect", NETWORK, RECOVERY_AGENT, check=False)
+    subprocess.run(  # noqa: S603 - fixed argv, no shell
+        [
+            "docker",  # noqa: S607
+            "compose",
+            "-p",
+            "modelforge",
+            "-f",
+            RECOVERY_COMPOSE,
+            "down",
+            "-v",
+        ],
+        capture_output=True,
+        text=True,
+        encoding="utf-8",
+        errors="replace",
+        timeout=180,
+        check=False,
+    )
+    with context.session() as session:
+        session.execute(
+            text(
+                "update compute_nodes set enabled = false, status = 'unavailable', "
+                "status_reason = 'M16 disposable chaos node; environment torn down' "
+                "where hostname = 'modelforge-chaos-rehearsal'"
+            )
+        )
+        session.execute(
+            text(
+                "update node_credentials set revoked_at = now() where revoked_at is null "
+                "and compute_node_id in "
+                "(select id from compute_nodes where hostname = 'modelforge-chaos-rehearsal')"
+            )
+        )
+        session.commit()
+
+
+@scenario(
+    "node-agent-disconnect",
+    "A Node Agent loses its network and reconnects",
+    failure_class="DEPENDENCY",
+    subsystem="Node Agent",
+    expected="heartbeat stops, liveness degrades honestly, and reconnect keeps one identity",
+    cleanup=_stop_recovery_agent,
+)
+def node_agent_disconnect(context: ScenarioContext) -> None:
+    """Losing a node must degrade liveness honestly rather than invent readiness.
+
+    A disposable agent is used deliberately. GPU Node reaches the control plane over the LAN rather
+    than the Compose network, so disconnecting a container would not interrupt it — and revoking or
+    isolating the production node to prove a point is exactly what M16 forbids.
+    """
+
+    _stop_recovery_agent(context)
+
+    token = _issue_enrollment(context)
+    env = {
+        "MODELFORGE_RECOVERY_AGENT_TOKEN": token,
+        "MODELFORGE_RECOVERY_AGENT_URL": "http://api:8000",
+        "MODELFORGE_RECOVERY_AGENT_HOSTNAME": "modelforge-chaos-rehearsal",
+        "MODELFORGE_RECOVERY_AGENT_DISPLAY_NAME": "M16 chaos rehearsal node",
+    }
+    _compose_up(env)
+
+    if not wait_for(lambda: _recovery_agent_node(context) is not None, timeout=120, interval=3.0):
+        raise ChaosError("the disposable agent never enrolled")
+    node = _recovery_agent_node(context)
+    assert node is not None
+    node_id, _ = node
+    context.note("disposable_node_id", str(node_id))
+
+    if not wait_for(
+        lambda: _heartbeat_moved(context, node_id, None), timeout=90, interval=3.0
+    ):
+        raise ChaosError("the disposable agent never published a heartbeat")
+    with context.session() as session:
+        before = session.execute(
+            text("select liveness_state, last_heartbeat_at from compute_nodes where id = :i"),
+            {"i": node_id},
+        ).first()
+    context.note("liveness_before", before[0])
+    heartbeat_before = before[1]
+
+    docker("network", "disconnect", NETWORK, RECOVERY_AGENT)
+    context.note("disconnected", True)
+    try:
+        # Longer than the configured offline threshold so liveness has to move, not just age.
+        time.sleep(100)
+        with context.session() as session:
+            during = session.execute(
+                text("select liveness_state, last_heartbeat_at from compute_nodes where id = :i"),
+                {"i": node_id},
+            ).first()
+        context.note("liveness_during", during[0])
+        advanced = during[1] != heartbeat_before
+        context.note("heartbeat_advanced_during_outage", advanced)
+        if advanced:
+            raise ChaosError("the heartbeat advanced while the agent was disconnected")
+        if during[0] == "online":
+            raise ChaosError("liveness still reported online after the agent went away")
+    finally:
+        docker("network", "connect", NETWORK, RECOVERY_AGENT, check=False)
+
+    reconnected = wait_for(
+        lambda: _heartbeat_moved(context, node_id, heartbeat_before), timeout=150, interval=3.0
+    )
+    context.note("reconnected", reconnected)
+    with context.session() as session:
+        after = session.execute(
+            text("select liveness_state from compute_nodes where id = :i"), {"i": node_id}
+        ).scalar_one()
+        identities = session.execute(
+            text(
+                "select count(*) from compute_nodes "
+                "where hostname = 'modelforge-chaos-rehearsal' and enabled"
+            )
+        ).scalar_one()
+    context.note("liveness_after", after)
+    context.note("enabled_identities_after", int(identities))
+    if int(identities) != 1:
+        raise ChaosError("the disconnect produced a duplicate node identity")
+    if not reconnected:
+        raise ChaosError("the Node Agent never resumed publishing")
+
+
+def _issue_enrollment(context: ScenarioContext) -> str:
+    payload = json.dumps(
+        {
+            "expires_in_seconds": 3600,
+            "display_name": "M16 chaos rehearsal node",
+            "role": "rehearsal",
+            "labels": {"rehearsal": "m16-chaos"},
+            "production_eligible": False,
+            "lab_eligible": False,
+            "benchmark_eligible": False,
+        }
+    ).encode()
+    request = urllib.request.Request(  # noqa: S310 - fixed http scheme
+        f"{context.base_url}/api/v1/admin/node-enrollments", data=payload, method="POST"
+    )
+    request.add_header("Content-Type", "application/json")
+    request.add_header("X-ModelForge-Admin-Token", context.token)
+    with urllib.request.urlopen(request, timeout=60) as response:  # noqa: S310
+        return str(json.loads(response.read().decode())["enrollment_token"])
+
+
+def _compose_up(extra_env: dict[str, str]) -> None:
+    import os
+
+    environment = {**os.environ, **extra_env}
+    completed = subprocess.run(  # noqa: S603 - fixed argv, no shell
+        [
+            "docker",  # noqa: S607
+            "compose",
+            "-p",
+            "modelforge",
+            "-f",
+            RECOVERY_COMPOSE,
+            "up",
+            "-d",
+            "--build",
+            "node-recovery-agent",
+        ],
+        capture_output=True,
+        text=True,
+        encoding="utf-8",
+        errors="replace",
+        timeout=900,
+        env=environment,
+        check=False,
+    )
+    if completed.returncode != 0:
+        raise ChaosError(f"the disposable agent did not start: {completed.stderr.strip()[:400]}")
+
+
+def _heartbeat_moved(context: ScenarioContext, node_id: Any, before: Any) -> bool:
+    with context.session() as session:
+        current = session.execute(
+            text("select last_heartbeat_at from compute_nodes where id = :i"), {"i": node_id}
+        ).scalar()
+    return current is not None and current != before
+
+
+@scenario(
+    "control-plane-restart-storm",
+    "The control plane restarts repeatedly under load",
+    failure_class="TRANSIENT",
+    subsystem="Control Plane",
+    expected="each start reconciles idempotently and no duplicate authoritative object appears",
+    recover=_restore_api,
+)
+def control_plane_restart_storm(context: ScenarioContext) -> None:
+    """Restart reconciliation must be idempotent, not merely survivable."""
+
+    with context.session() as session:
+        before = _authoritative_counts(session)
+    context.note("counts_before", before)
+
+    restarts = 4
+    statuses: list[int] = []
+    for index in range(restarts):
+        docker("restart", API)
+        healthy = wait_for(lambda: _api_healthy(context), timeout=120)
+        if not healthy:
+            raise ChaosError(f"the control plane did not return after restart {index + 1}")
+        statuses.append(http(f"{context.base_url}/api/v1/models").status)
+    context.note("restarts", restarts)
+    context.note("registry_status_after_each_restart", statuses)
+
+    with context.session() as session:
+        after = _authoritative_counts(session)
+    context.note("counts_after", after)
+
+    drifted = {
+        key: (before[key], after[key])
+        for key in before
+        if key not in {"audit_events"} and after[key] != before[key]
+    }
+    context.note("authoritative_drift", drifted)
+    if drifted:
+        raise ChaosError(f"restart storm changed authoritative state: {drifted}")
+
+
+def _authoritative_counts(session: Session) -> dict[str, int]:
+    tables = (
+        "models",
+        "model_revisions",
+        "model_artifacts",
+        "artifact_sets",
+        "projects",
+        "project_bindings",
+        "capability_deployments",
+        "lifecycle_operations",
+        "migration_plans",
+        "migration_cutover_operations",
+        "compute_nodes",
+        "node_credentials",
+        "service_clients",
+        "backup_sets",
+        "audit_events",
+    )
+    counts: dict[str, int] = {}
+    for table in tables:
+        counts[table] = int(
+            session.execute(text(f"select count(*) from {table}")).scalar_one()  # noqa: S608
+        )
+    return counts
+
+
+@scenario(
+    "invalid-auth-burst",
+    "A burst of invalid operator credentials",
+    failure_class="AUTH",
+    subsystem="Control Plane",
+    expected="every attempt is refused identically, with no amplification and no enumeration signal",
+)
+def invalid_auth_burst(context: ScenarioContext) -> None:
+    """A wrong credential must not reveal whether it was close to right."""
+
+    attempts = 60
+    tokens = [uuid.UUID(int=context.rng.getrandbits(128)).hex for _ in range(attempts)]
+    tokens[0] = ""
+    tokens[1] = "x"
+    tokens[2] = context.token[:-1] + ("a" if context.token[-1] != "a" else "b")
+    statuses: list[int] = []
+    bodies: set[str] = set()
+    latencies: list[float] = []
+    for token in tokens:
+        probe = http(
+            f"{context.base_url}/api/v1/admin/recovery/dashboard",
+            token=token or None,
+            timeout=10.0,
+        )
+        statuses.append(probe.status)
+        bodies.add(probe.body[:0])
+        latencies.append(probe.latency_ms)
+    context.note("attempts", attempts)
+    context.note("distinct_statuses", sorted(set(statuses)))
+    context.note("max_latency_ms", round(max(latencies), 2))
+    context.note("mean_latency_ms", round(sum(latencies) / len(latencies), 2))
+
+    if set(statuses) != {401}:
+        raise ChaosError(f"invalid credentials produced varying statuses: {sorted(set(statuses))}")
+
+    good = http(f"{context.base_url}/api/v1/admin/recovery/dashboard", token=context.token)
+    context.note("valid_credential_still_works", good.ok)
+    if not good.ok:
+        raise ChaosError("a burst of invalid credentials locked out the valid one")
+
+
+# The backup root is a named volume, not a path inside the API container. Injecting this fault used
+# to be `docker exec -u 0 modelforge-api-1 chmod ...`, which stopped working the moment M16 hardened
+# the API container: with all capabilities dropped and no-new-privileges set, even uid 0 inside that
+# container cannot change a mode. That is the hardening behaving correctly, so the fault moves
+# outside the product instead of the hardening being weakened to keep a test convenient.
+#
+# A disposable Alpine container mounts the same volume and changes the mode there. It touches only
+# the isolated backup volume, never a host path, and is removed on exit.
+BACKUP_VOLUME = "modelforge_modelforge-backups"
+
+
+def _set_backup_root_mode(mode: str, *, check: bool = True) -> None:
+    docker(
+        "run",
+        "--rm",
+        "-v",
+        f"{BACKUP_VOLUME}:/data/backups",
+        "alpine:3.20",
+        "chmod",
+        mode,
+        "/data/backups",
+        check=check,
+    )
+
+
+@scenario(
+    "backup-destination-unavailable",
+    "The backup destination cannot be written",
+    failure_class="CAPACITY",
+    subsystem="Backup storage",
+    expected="the backup fails closed and serving is unaffected",
+)
+def backup_destination_unavailable(context: ScenarioContext) -> None:
+    """A backup that cannot be written must fail, not silently produce a partial recovery point."""
+
+    backup_id = f"chaos-unwritable-{context.seed:08d}"
+    _set_backup_root_mode("0555")
+    try:
+        payload = json.dumps(
+            {
+                "backup_id": backup_id,
+                "reason": "M16 chaos scenario: the backup root is not writable",
+            }
+        ).encode()
+        request = urllib.request.Request(  # noqa: S310 - fixed http scheme
+            f"{context.base_url}/api/v1/admin/recovery/backups", data=payload, method="POST"
+        )
+        request.add_header("Content-Type", "application/json")
+        request.add_header("X-ModelForge-Admin-Token", context.token)
+        status = 0
+        try:
+            with urllib.request.urlopen(request, timeout=120) as response:  # noqa: S310
+                status = response.status
+                body = json.loads(response.read().decode())
+        except urllib.error.HTTPError as error:
+            status = error.code
+            body = {}
+        context.note("backup_http_status", status)
+        context.note("backup_state", body.get("state"))
+        context.note("backup_failure_code", body.get("failure_code"))
+    finally:
+        _set_backup_root_mode("0755", check=False)
+
+    serving = http(f"{context.base_url}/api/v1/health/ready")
+    context.note("serving_unaffected", serving.ok)
+
+    with context.session() as session:
+        state = session.execute(
+            text("select state from backup_sets where backup_id = :b"), {"b": backup_id}
+        ).scalar()
+    context.note("persisted_backup_state", state)
+    if state == "VERIFIED":
+        raise ChaosError("an unwritable destination still produced a verified backup")
+    if not serving.ok:
+        raise ChaosError("a failed backup degraded serving")
+
+
+def _cleanup_chaos_backups(context: ScenarioContext) -> None:
+    with context.session() as session:
+        session.execute(
+            text("delete from backup_manifest_entries where backup_set_id in "
+                 "(select id from backup_sets where backup_id like 'chaos-%')")
+        )
+        session.execute(text("delete from backup_sets where backup_id like 'chaos-%'"))
+        session.commit()
+
+
+SCENARIOS["backup-destination-unavailable"].cleanup = _cleanup_chaos_backups
+
+
+# --------------------------------------------------------------------- runner
+
+
+def run_scenario(scenario_obj: Scenario, context: ScenarioContext) -> dict[str, Any]:
+    started = datetime.now(UTC)
+    record: dict[str, Any] = {
+        "scenario": scenario_obj.key,
+        "title": scenario_obj.title,
+        "failure_class": scenario_obj.failure_class,
+        "subsystem": scenario_obj.subsystem,
+        "expected": scenario_obj.expected,
+        "seed": context.seed,
+        "started_at": started.isoformat(),
+    }
+
+    with context.session() as session:
+        baseline = check_invariants(session)
+    record["invariants_before"] = {
+        "checked": baseline.checked,
+        "violated": baseline.violated,
+        "violations": [
+            item.key for item in baseline.results if item.status is InvariantStatus.VIOLATED
+        ],
+    }
+    if baseline.violated:
+        record["outcome"] = "PREFLIGHT_FAILED"
+        record["detail"] = "invariants were already violated before injection"
+        record["finished_at"] = datetime.now(UTC).isoformat()
+        return record
+
+    outcome = "PASSED"
+    detail = ""
+    try:
+        scenario_obj.run(context)
+    except ChaosError as error:
+        outcome = "FAILED"
+        detail = str(error)
+    except Exception as error:  # noqa: BLE001 - the harness must report, not crash the run
+        outcome = "HARNESS_ERROR"
+        detail = f"{type(error).__name__}: {error}"
+
+    if scenario_obj.recover is not None:
+        try:
+            scenario_obj.recover(context)
+        except ChaosError as error:
+            outcome = "RECOVERY_FAILED"
+            detail = f"{detail} | recovery: {error}".strip(" |")
+
+    if scenario_obj.cleanup is not None:
+        try:
+            scenario_obj.cleanup(context)
+            record["cleanup"] = "DONE"
+        except Exception as error:  # noqa: BLE001
+            record["cleanup"] = f"FAILED: {type(error).__name__}: {error}"
+            outcome = "CLEANUP_FAILED" if outcome == "PASSED" else outcome
+    else:
+        record["cleanup"] = "NOT_REQUIRED"
+
+    with context.session() as session:
+        after = check_invariants(session)
+    record["invariants_after"] = {
+        "checked": after.checked,
+        "violated": after.violated,
+        "violations": [
+            {"key": item.key, "examples": item.violations}
+            for item in after.results
+            if item.status is InvariantStatus.VIOLATED
+        ],
+    }
+    if after.violated and outcome == "PASSED":
+        outcome = "INVARIANT_VIOLATED"
+        detail = "the platform survived the fault but broke an invariant"
+
+    record["observations"] = context.observations
+    record["outcome"] = outcome
+    record["detail"] = detail
+    record["finished_at"] = datetime.now(UTC).isoformat()
+    record["duration_seconds"] = round(
+        (datetime.fromisoformat(record["finished_at"]) - started).total_seconds(), 3
+    )
+    return record
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--base-url", default=DEFAULT_BASE_URL)
+    parser.add_argument("--database-url", default=DEFAULT_DATABASE_URL)
+    parser.add_argument("--token", default=None, help="operator API key")
+    parser.add_argument("--scenario", action="append", default=[])
+    parser.add_argument("--all", action="store_true")
+    parser.add_argument("--list", action="store_true")
+    parser.add_argument("--seed", type=int, default=None)
+    parser.add_argument("--report", default=None)
+    args = parser.parse_args(argv)
+
+    if args.list:
+        for key, item in sorted(SCENARIOS.items()):
+            print(f"{key:32} {item.failure_class:12} {item.subsystem:16} {item.title}")
+        return 0
+    if not args.database_url:
+        parser.error(
+            "--database-url or MODELFORGE_RUNTIME_DATABASE_URL is required "
+            "(use the non-owner runtime role)"
+        )
+
+    keys = list(SCENARIOS) if args.all else args.scenario
+    if not keys:
+        parser.error("choose --scenario, --all or --list")
+    unknown = [key for key in keys if key not in SCENARIOS]
+    if unknown:
+        parser.error(f"unknown scenarios: {unknown}")
+
+    token = args.token
+    if not token:
+        env = Path(__file__).resolve().parents[1] / ".env"
+        if env.is_file():
+            for line in env.read_text("utf-8").splitlines():
+                if line.startswith("MODELFORGE_OPERATOR_API_KEY="):
+                    token = line.split("=", 1)[1].strip()
+    if not token:
+        parser.error("an operator API key is required")
+
+    seed = args.seed if args.seed is not None else random.randrange(1, 10**8)
+    records = []
+    for key in keys:
+        context = ScenarioContext(
+            base_url=args.base_url,
+            database_url=args.database_url,
+            token=token,
+            seed=seed,
+            rng=random.Random(seed),
+        )
+        print(f"--- {key} (seed {seed}) ---", flush=True)
+        record = run_scenario(SCENARIOS[key], context)
+        records.append(record)
+        print(f"    {record['outcome']}  {record.get('detail', '')}", flush=True)
+        for name, value in record["observations"].items():
+            print(f"      {name}: {value}", flush=True)
+
+    report = {
+        "generated_at": datetime.now(UTC).isoformat(),
+        "seed": seed,
+        "scenarios": records,
+        "passed": sum(item["outcome"] == "PASSED" for item in records),
+        "total": len(records),
+    }
+    if args.report:
+        Path(args.report).write_text(json.dumps(report, indent=2), encoding="utf-8")
+        print(f"report written to {args.report}")
+    return 0 if report["passed"] == report["total"] else 1
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/m16_invariants.py

@@ -0,0 +1,56 @@
+"""Run the M16 platform invariants against a ModelForge database and report each outcome.
+
+Read-only. Used as a standalone operator check, and by the chaos harness before injection, after
+injection and after recovery, so a scenario can prove it left no invariant broken.
+
+    python scripts/m16_invariants.py --database-url postgresql+psycopg://... [--json]
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from pathlib import Path
+
+from sqlalchemy import create_engine
+from sqlalchemy.orm import Session
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "backend" / "src"))
+
+from modelforge_api.services.invariants import (  # noqa: E402
+    InvariantStatus,
+    check_invariants,
+)
+from modelforge_api.settings import get_settings  # noqa: E402
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--database-url", default=None)
+    parser.add_argument("--json", action="store_true", help="emit the full machine-readable report")
+    parser.add_argument("--label", default="invariants", help="label used in the human summary")
+    args = parser.parse_args(argv)
+
+    url = args.database_url or get_settings().database_url
+    engine = create_engine(url, pool_pre_ping=True)
+    try:
+        with Session(engine) as session:
+            report = check_invariants(session)
+    finally:
+        engine.dispose()
+
+    if args.json:
+        print(report.model_dump_json(indent=2))
+    else:
+        print(f"{args.label}: {report.holding}/{report.checked} hold, {report.violated} violated")
+        for item in report.results:
+            mark = "OK  " if item.status is InvariantStatus.HOLDS else "FAIL"
+            print(f"  {mark} {item.key:38} {item.detail}")
+            for example in item.violations:
+                print(f"         - {example}")
+    return 0 if report.ok else 1
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/m16_release_gate.py

@@ -0,0 +1,317 @@
+"""M16 v1 technical release gate.
+
+A deterministic aggregation of the evidence M16 produces. It reads results rather than generating
+them: the chaos, soak and security runs happen first and this decides what they add up to.
+
+The verdict is mechanical on purpose. "Looks fine" is not a release decision, and a gate that can
+be argued with is a gate that will be.
+
+    python scripts/m16_release_gate.py --chaos chaos.json --soak soak.json --report gate.json
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import sys
+from dataclasses import dataclass, field
+from datetime import UTC, datetime
+from pathlib import Path
+from typing import Any
+
+from sqlalchemy import create_engine, text
+from sqlalchemy.orm import Session
+
+ROOT = Path(__file__).resolve().parents[1]
+sys.path.insert(0, str(ROOT / "backend" / "src"))
+
+from modelforge_api.services.invariants import (  # noqa: E402
+    InvariantStatus,
+    check_invariants,
+)
+
+DEFAULT_DATABASE_URL = os.getenv("MODELFORGE_RUNTIME_DATABASE_URL")
+
+# A finding in any of these classes blocks the release outright. Everything else is a documented
+# limitation at most.
+HARD_BLOCKERS = (
+    "data corruption",
+    "privilege escalation",
+    "secret leakage",
+    "unbounded resource exhaustion",
+    "unrecoverable control-plane crash",
+    "duplicate production state",
+    "unsafe credential reuse",
+    "artifact integrity bypass",
+    "production alias or recognizer mutation",
+    "critical exploitable vulnerability",
+)
+
+
+@dataclass
+class Check:
+    name: str
+    passed: bool
+    detail: str
+    blocking: bool = True
+    evidence: dict[str, Any] = field(default_factory=dict)
+
+
+def _load(path: str | None) -> dict[str, Any] | None:
+    if not path:
+        return None
+    candidate = Path(path)
+    if not candidate.is_file():
+        return None
+    try:
+        return dict(json.loads(candidate.read_text("utf-8")))
+    except (json.JSONDecodeError, ValueError):
+        return None
+
+
+def invariant_check(database_url: str) -> Check:
+    engine = create_engine(database_url, pool_pre_ping=True)
+    try:
+        with Session(engine) as session:
+            report = check_invariants(session)
+    finally:
+        engine.dispose()
+    violations = [
+        item.key for item in report.results if item.status is InvariantStatus.VIOLATED
+    ]
+    return Check(
+        name="platform invariants",
+        passed=report.violated == 0,
+        detail=f"{report.holding}/{report.checked} hold",
+        evidence={"violations": violations},
+    )
+
+
+def chaos_check(report: dict[str, Any] | None) -> Check:
+    if report is None:
+        return Check("chaos scenarios", False, "no chaos report supplied")
+    scenarios = report.get("scenarios", [])
+    failed = [item["scenario"] for item in scenarios if item.get("outcome") != "PASSED"]
+    unclean = [
+        item["scenario"]
+        for item in scenarios
+        if item.get("cleanup", "NOT_REQUIRED") not in ("DONE", "NOT_REQUIRED")
+    ]
+    return Check(
+        name="chaos scenarios",
+        passed=not failed and not unclean,
+        detail=f"{len(scenarios) - len(failed)}/{len(scenarios)} passed",
+        evidence={"failed": failed, "unclean_cleanup": unclean, "seed": report.get("seed")},
+    )
+
+
+def soak_check(report: dict[str, Any] | None, minimum_minutes: float) -> Check:
+    if report is None:
+        return Check("soak", False, "no soak report supplied")
+    duration = float(report.get("actual_duration_seconds", 0)) / 60
+    after = report.get("resources_after", {})
+    leases = int(after.get("active_leases", -1))
+    in_flight = int(after.get("serving_jobs_in_flight", -1))
+    violated = int(report.get("invariants_final", {}).get("violated", 1))
+    internal = sum(
+        int(entry.get("internal_errors", 0)) for entry in report.get("by_kind", {}).values()
+    )
+    passed = (
+        duration >= minimum_minutes
+        and leases == 0
+        and in_flight == 0
+        and violated == 0
+    )
+    return Check(
+        name="soak",
+        passed=passed,
+        detail=(
+            f"{duration:.1f} min, {report.get('total_requests', 0)} requests, "
+            f"{leases} active leases, {in_flight} jobs in flight, {internal} internal errors"
+        ),
+        evidence={
+            "duration_minutes": round(duration, 2),
+            "minimum_minutes": minimum_minutes,
+            "total_requests": report.get("total_requests"),
+            "internal_errors": internal,
+            "invariants_violated": violated,
+            "row_growth": report.get("row_growth"),
+        },
+    )
+
+
+def orphan_check(database_url: str) -> Check:
+    engine = create_engine(database_url, pool_pre_ping=True)
+    try:
+        with Session(engine) as session:
+            leases = int(
+                session.execute(
+                    text(
+                        "select count(*) from serving_gpu_leases where released_at is null "
+                        "and state in ('pending','granted','active','held')"
+                    )
+                ).scalar_one()
+            )
+            jobs = int(
+                session.execute(
+                    text(
+                        "select count(*) from serving_jobs "
+                        "where status in ('queued','leased','running')"
+                    )
+                ).scalar_one()
+            )
+    finally:
+        engine.dispose()
+    return Check(
+        name="no orphaned work",
+        passed=leases == 0 and jobs == 0,
+        detail=f"{leases} active leases, {jobs} jobs in flight",
+        evidence={"active_leases": leases, "serving_jobs_in_flight": jobs},
+    )
+
+
+def alert_check(database_url: str) -> Check:
+    """Artificial alerts raised during the gate must not be left firing."""
+
+    engine = create_engine(database_url, pool_pre_ping=True)
+    try:
+        with Session(engine) as session:
+            rows = session.execute(
+                text(
+                    "select alert_type, state, count(*) from operational_alerts "
+                    "where state in ('PENDING','FIRING') group by alert_type, state"
+                )
+            ).all()
+    finally:
+        engine.dispose()
+    active = [
+        {"alert_type": alert_type, "state": state, "count": int(count)}
+        for alert_type, state, count in rows
+    ]
+    return Check(
+        name="no unresolved artificial alerts",
+        passed=not active,
+        detail=f"{sum(item['count'] for item in active)} alerts pending or firing",
+        blocking=False,
+        evidence={"active": active},
+    )
+
+
+def production_identity_check(database_url: str, expected: Path | None) -> Check:
+    engine = create_engine(database_url, pool_pre_ping=True)
+    try:
+        with Session(engine) as session:
+            rows = session.execute(
+                text(
+                    "select d.id::text, c.key, d.status, d.artifact_set_id::text, "
+                    "d.runtime_profile_id::text, d.capability_contract_id::text "
+                    "from capability_deployments d "
+                    "join capability_contracts cc on cc.id = d.capability_contract_id "
+                    "join capabilities c on c.id = cc.capability_id "
+                    "where d.production order by c.key, d.id"
+                )
+            ).all()
+    finally:
+        engine.dispose()
+    current = ["|".join(str(value) for value in row) for row in rows]
+    if expected is None or not expected.is_file():
+        return Check(
+            name="production identities unchanged",
+            passed=False,
+            detail="no baseline supplied to compare against",
+            evidence={"current": current},
+        )
+    baseline = [
+        line.strip() for line in expected.read_text("utf-8").splitlines() if line.strip()
+    ]
+    added = sorted(set(current) - set(baseline))
+    removed = sorted(set(baseline) - set(current))
+    return Check(
+        name="production identities unchanged",
+        passed=not added and not removed,
+        detail=f"{len(current)} production deployments",
+        evidence={"added": added, "removed": removed},
+    )
+
+
+def sbom_check() -> Check:
+    sbom = ROOT / "docs" / "security" / "sbom" / "modelforge-cyclonedx.json"
+    provenance = ROOT / "docs" / "security" / "sbom" / "image-provenance.json"
+    if not sbom.is_file() or not provenance.is_file():
+        return Check("supply-chain inventory", False, "SBOM or provenance is missing")
+    document = json.loads(sbom.read_text("utf-8"))
+    images = json.loads(provenance.read_text("utf-8"))
+    components = len(document.get("components", []))
+    return Check(
+        name="supply-chain inventory",
+        passed=components > 0 and bool(images.get("images")),
+        detail=f"{components} components across {len(images.get('images', []))} images",
+        evidence={"source_commit": images.get("source_commit")},
+    )
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--database-url", default=DEFAULT_DATABASE_URL)
+    parser.add_argument("--chaos", default=None)
+    parser.add_argument("--soak", default=None)
+    parser.add_argument("--production-baseline", default=None)
+    parser.add_argument("--minimum-soak-minutes", type=float, default=30.0)
+    parser.add_argument("--report", default=None)
+    args = parser.parse_args(argv)
+    if not args.database_url:
+        parser.error(
+            "--database-url or MODELFORGE_RUNTIME_DATABASE_URL is required "
+            "(use the non-owner runtime role)"
+        )
+
+    checks = [
+        invariant_check(args.database_url),
+        chaos_check(_load(args.chaos)),
+        soak_check(_load(args.soak), args.minimum_soak_minutes),
+        orphan_check(args.database_url),
+        alert_check(args.database_url),
+        production_identity_check(
+            args.database_url,
+            Path(args.production_baseline) if args.production_baseline else None,
+        ),
+        sbom_check(),
+    ]
+
+    blocking_failures = [item for item in checks if item.blocking and not item.passed]
+    advisory_failures = [item for item in checks if not item.blocking and not item.passed]
+    readiness = "READY_FOR_PACKAGING" if not blocking_failures else "NOT_READY"
+
+    report = {
+        "generated_at": datetime.now(UTC).isoformat(),
+        "hard_blocker_classes": list(HARD_BLOCKERS),
+        "checks": [
+            {
+                "name": item.name,
+                "passed": item.passed,
+                "blocking": item.blocking,
+                "detail": item.detail,
+                "evidence": item.evidence,
+            }
+            for item in checks
+        ],
+        "blocking_failures": [item.name for item in blocking_failures],
+        "advisory_failures": [item.name for item in advisory_failures],
+        "v1_technical_release_readiness": readiness,
+    }
+
+    for item in checks:
+        mark = "PASS" if item.passed else ("FAIL" if item.blocking else "WARN")
+        print(f"  {mark}  {item.name:36} {item.detail}")
+    print()
+    print(f"V1_TECHNICAL_RELEASE_READINESS = {readiness}")
+
+    if args.report:
+        Path(args.report).write_text(json.dumps(report, indent=2), encoding="utf-8")
+        print(f"report written to {args.report}")
+    return 0 if readiness == "READY_FOR_PACKAGING" else 1
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/m16_sbom.py

@@ -0,0 +1,230 @@
+"""Generate a CycloneDX SBOM and an image provenance record for the ModelForge v1 components.
+
+The SBOM lists components and their versions, not their contents: a release inventory has to be
+readable and diffable, so no dependency binaries are stored. Provenance binds each built image to
+the exact source commit and image digest that produced it, which is what makes the SBOM verifiable
+rather than merely present.
+
+    python scripts/m16_sbom.py --output docs/security/sbom
+"""
+
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+import subprocess
+import sys
+from datetime import UTC, datetime
+from pathlib import Path
+from typing import Any
+
+ROOT = Path(__file__).resolve().parents[1]
+
+PYTHON_IMAGES = {
+    "modelforge-api": "backend",
+    "modelforge-node-agent": "node-agent",
+    "modelforge-runtime-worker": "runtime-worker",
+}
+NODE_COMPONENTS = {"modelforge-web": "frontend"}
+
+
+def run(*args: str, timeout: int = 300) -> str:
+    completed = subprocess.run(
+        list(args),
+        capture_output=True,
+        text=True,
+        timeout=timeout,
+        check=False,
+    )
+    return (completed.stdout or "").strip()
+
+
+def git(*args: str) -> str:
+    return run("git", "-C", str(ROOT), *args)
+
+
+def docker(*args: str, timeout: int = 300) -> str:
+    return run("docker", *args, timeout=timeout)
+
+
+def python_components(image: str) -> list[dict[str, Any]]:
+    """Read the installed distributions from the image itself, not from a manifest."""
+
+    raw = docker(
+        "run",
+        "--rm",
+        "--entrypoint",
+        "python",
+        image,
+        "-c",
+        "import json,importlib.metadata as m;"
+        "print(json.dumps(sorted(((d.metadata['Name'] or '?'), (d.version or '?')) "
+        "for d in m.distributions())))",
+        timeout=600,
+    )
+    try:
+        entries = json.loads(raw)
+    except json.JSONDecodeError:
+        return []
+    seen: set[tuple[str, str]] = set()
+    components = []
+    for name, version in entries:
+        key = (str(name).lower(), str(version))
+        if key in seen:
+            continue
+        seen.add(key)
+        components.append(
+            {
+                "type": "library",
+                "name": str(name),
+                "version": str(version),
+                "purl": f"pkg:pypi/{str(name).lower()}@{version}",
+                "scope": "required",
+            }
+        )
+    return components
+
+
+def node_components(directory: Path) -> list[dict[str, Any]]:
+    lock = directory / "package-lock.json"
+    if not lock.is_file():
+        return []
+    data = json.loads(lock.read_text(encoding="utf-8"))
+    components = []
+    for path, entry in sorted(data.get("packages", {}).items()):
+        if not path.startswith("node_modules/"):
+            continue
+        name = path.removeprefix("node_modules/")
+        version = entry.get("version")
+        if not version:
+            continue
+        components.append(
+            {
+                "type": "library",
+                "name": name,
+                "version": str(version),
+                "purl": f"pkg:npm/{name}@{version}",
+                "scope": "optional" if entry.get("dev") else "required",
+            }
+        )
+    return components
+
+
+def image_provenance(image: str, commit: str) -> dict[str, Any]:
+    digest = docker("inspect", "--format", "{{index .RepoDigests 0}}", image) or None
+    labels_raw = docker("inspect", "--format", "{{json .Config.Labels}}", image)
+    try:
+        labels = json.loads(labels_raw or "null") or {}
+    except json.JSONDecodeError:
+        labels = {}
+    if not isinstance(labels, dict):
+        labels = {}
+    return {
+        "image": image,
+        "image_id": docker("inspect", "--format", "{{.Id}}", image),
+        "repo_digest": digest,
+        "created": docker("inspect", "--format", "{{.Created}}", image),
+        "base_image": docker("inspect", "--format", "{{index .Config.Labels \"base\"}}", image)
+        or None,
+        "source_commit": commit,
+        "source_repository": git("config", "--get", "remote.origin.url"),
+        "oci": {
+            key: value
+            for key, value in labels.items()
+            if key.startswith("org.opencontainers.image.")
+        },
+    }
+
+
+def build(
+    output: Path,
+    *,
+    version: str,
+    source_commit: str | None = None,
+    generated_at: str | None = None,
+) -> dict[str, Any]:
+    commit = source_commit or git("rev-parse", "HEAD")
+    generated = generated_at or datetime.now(UTC).isoformat()
+    components: list[dict[str, Any]] = []
+    provenance: list[dict[str, Any]] = []
+
+    for image_name in PYTHON_IMAGES:
+        image = f"{image_name}:{version}"
+        entries = python_components(image)
+        for entry in entries:
+            entry["properties"] = [{"name": "modelforge:image", "value": image}]
+        components.extend(entries)
+        provenance.append(image_provenance(image, commit))
+
+    for image_name, directory in NODE_COMPONENTS.items():
+        image = f"{image_name}:{version}"
+        entries = node_components(ROOT / directory)
+        for entry in entries:
+            entry["properties"] = [{"name": "modelforge:image", "value": image}]
+        components.extend(entries)
+        provenance.append(image_provenance(image, commit))
+
+    serial = hashlib.sha256(
+        json.dumps([c["purl"] for c in components], sort_keys=True).encode()
+    ).hexdigest()
+
+    sbom = {
+        "bomFormat": "CycloneDX",
+        "specVersion": "1.5",
+        "serialNumber": f"urn:uuid:{serial[:8]}-{serial[8:12]}-{serial[12:16]}-{serial[16:20]}-{serial[20:32]}",
+        "version": 1,
+        "metadata": {
+            "timestamp": generated,
+            "component": {
+                "type": "application",
+                "name": "itworx-modelforge",
+                "version": version,
+            },
+            "properties": [
+                {"name": "modelforge:source_commit", "value": commit},
+                {"name": "modelforge:generated_by", "value": "scripts/m16_sbom.py"},
+            ],
+        },
+        "components": components,
+    }
+
+    output.mkdir(parents=True, exist_ok=True)
+    (output / "modelforge-cyclonedx.json").write_text(
+        json.dumps(sbom, indent=2) + "\n", encoding="utf-8"
+    )
+    (output / "image-provenance.json").write_text(
+        json.dumps(
+            {"generated_at": generated, "source_commit": commit, "images": provenance},
+            indent=2,
+        )
+        + "\n",
+        encoding="utf-8",
+    )
+    return {
+        "components": len(components),
+        "images": len(provenance),
+        "source_commit": commit,
+        "output": str(output),
+    }
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--output", default=str(ROOT / "docs" / "security" / "sbom"))
+    parser.add_argument("--version", default=(ROOT / "VERSION").read_text("utf-8").strip())
+    parser.add_argument("--source-commit", default=None)
+    parser.add_argument("--generated-at", default=None)
+    args = parser.parse_args(argv)
+    summary = build(
+        Path(args.output),
+        version=args.version,
+        source_commit=args.source_commit,
+        generated_at=args.generated_at,
+    )
+    print(json.dumps(summary, indent=2))
+    return 0 if summary["components"] else 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())

scripts/m16_soak.py

@@ -0,0 +1,667 @@
+"""M16 bounded soak.
+
+A realistic mixed workload held for a bounded period, measuring what actually drifts: latency
+distribution, cold versus warm execution, memory, GPU, database growth, queue depth, leases and
+orphaned work. Every number reported here is measured; nothing is extrapolated from a short run.
+
+The workload deliberately mixes bursts, steady state and idle periods. A soak that only hammers
+never exercises the keep-warm and unload paths, and a soak that only idles never exercises
+admission control.
+
+Production is read, never mutated: no promotion, no alias change, no external workload is touched.
+The only writes are gateway requests through a disposable LAB client and the operational polls the
+platform already performs for itself.
+
+    python scripts/m16_soak.py --minutes 45 --report soak.json
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import random
+import socket
+import statistics
+import subprocess  # noqa: S404 - fixed argv container inspection, never a shell
+import sys
+import threading
+import time
+import urllib.error
+import urllib.parse
+import urllib.request
+from collections import Counter
+from dataclasses import dataclass, field
+from datetime import UTC, datetime
+from pathlib import Path
+from typing import Any
+
+from sqlalchemy import create_engine, text
+from sqlalchemy.orm import Session
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "backend" / "src"))
+
+from modelforge_api.services.invariants import (  # noqa: E402
+    InvariantStatus,
+    check_invariants,
+)
+
+DEFAULT_BASE_URL = "http://127.0.0.1:8000"
+DEFAULT_DATABASE_URL = os.getenv("MODELFORGE_RUNTIME_DATABASE_URL")
+
+SENTENCES = [
+    "ModelForge binds applications to capabilities rather than to model paths.",
+    "A verified backup is the only backup that counts as a recovery point.",
+    "Stale telemetry must never be presented as present-day truth.",
+    "Every production promotion carries evidence from local validation.",
+    "Downloaded model repositories stay untrusted until supply-chain checks complete.",
+    "The scheduler refuses unsafe work instead of risking an out-of-memory failure.",
+    "Recovery reconciliation clears current truth and re-measures it.",
+    "An embedding-space change never silently reuses incompatible vectors.",
+]
+
+
+@dataclass
+class Sample:
+    at: float
+    kind: str
+    ok: bool
+    status: int
+    latency_ms: float
+    detail: dict[str, Any] = field(default_factory=dict)
+
+
+@dataclass
+class SoakState:
+    samples: list[Sample] = field(default_factory=list)
+    lock: threading.Lock = field(default_factory=threading.Lock)
+    stop: threading.Event = field(default_factory=threading.Event)
+
+    def record(self, sample: Sample) -> None:
+        with self.lock:
+            self.samples.append(sample)
+
+
+def docker(*args: str) -> str:
+    try:
+        completed = subprocess.run(  # noqa: S603 - fixed argv, no shell
+            ["docker", *args],  # noqa: S607
+            capture_output=True,
+            text=True,
+            encoding="utf-8",
+            errors="replace",
+            timeout=60,
+            check=False,
+        )
+    except (OSError, subprocess.TimeoutExpired):
+        return ""
+    return (completed.stdout or "").strip()
+
+
+def container_stats(names: list[str]) -> dict[str, dict[str, Any]]:
+    """Point-in-time RSS and CPU per container, read from the runtime rather than guessed."""
+
+    raw = docker(
+        "stats",
+        "--no-stream",
+        "--format",
+        "{{.Name}}\t{{.MemUsage}}\t{{.CPUPerc}}\t{{.PIDs}}",
+        *names,
+    )
+    result: dict[str, dict[str, Any]] = {}
+    for line in raw.splitlines():
+        parts = line.split("\t")
+        if len(parts) != 4:
+            continue
+        name, memory, cpu, pids = parts
+        used = memory.split("/")[0].strip()
+        result[name] = {"memory": used, "cpu_percent": cpu.strip(), "pids": pids.strip()}
+    return result
+
+
+def restart_counts(names: list[str]) -> dict[str, int]:
+    counts: dict[str, int] = {}
+    for name in names:
+        value = docker("inspect", "--format", "{{.RestartCount}}", name)
+        counts[name] = int(value) if value.isdigit() else -1
+    return counts
+
+
+def http(
+    url: str,
+    *,
+    token: str | None = None,
+    bearer: str | None = None,
+    method: str = "GET",
+    payload: dict[str, Any] | None = None,
+    timeout: float = 180.0,
+) -> tuple[bool, int, float, dict[str, Any]]:
+    body = json.dumps(payload).encode() if payload is not None else None
+    request = urllib.request.Request(url, data=body, method=method)  # noqa: S310 - fixed scheme
+    if token:
+        request.add_header("X-ModelForge-Admin-Token", token)
+    if bearer:
+        request.add_header("Authorization", f"Bearer {bearer}")
+    if body is not None:
+        request.add_header("Content-Type", "application/json")
+    started = time.perf_counter()
+    try:
+        with urllib.request.urlopen(request, timeout=timeout) as response:  # noqa: S310
+            raw = response.read()
+            elapsed = (time.perf_counter() - started) * 1000
+            try:
+                decoded = json.loads(raw.decode("utf-8"))
+            except (UnicodeDecodeError, json.JSONDecodeError):
+                decoded = {}
+            return True, response.status, elapsed, decoded if isinstance(decoded, dict) else {}
+    except urllib.error.HTTPError as error:
+        elapsed = (time.perf_counter() - started) * 1000
+        try:
+            decoded = json.loads(error.read().decode("utf-8"))
+        except Exception:  # noqa: BLE001 - an unreadable error body is still an error
+            decoded = {}
+        return False, error.code, elapsed, decoded if isinstance(decoded, dict) else {}
+    except (urllib.error.URLError, TimeoutError, OSError) as error:
+        elapsed = (time.perf_counter() - started) * 1000
+        return False, 0, elapsed, {"transport_error": type(error).__name__}
+
+
+# The soak reports latency percentiles, so the client-side floor has to be negligible before any
+# of them mean anything. A first run measured a p50 of two seconds on every route including
+# /health/live; the cause was the harness resolving "localhost" to ::1 on Windows and waiting out
+# the connect timeout before falling back to IPv4. That inflated every percentile by a constant
+# ~2,045 ms of client cost and said nothing about the platform. Measure the floor and refuse rather
+# than publish a number that is mostly the measuring instrument.
+MAX_CONNECT_FLOOR_MS = 100.0
+
+
+def connect_floor_ms(base_url: str, samples: int = 7) -> float:
+    """Median TCP connect time to the target, excluding anything the platform does."""
+
+    parsed = urllib.parse.urlparse(base_url)
+    host = parsed.hostname or "127.0.0.1"
+    port = parsed.port or (443 if parsed.scheme == "https" else 80)
+    timings = []
+    for _ in range(samples):
+        started = time.perf_counter()
+        try:
+            connection = socket.create_connection((host, port), timeout=10)
+        except OSError:
+            return float("inf")
+        connection.close()
+        timings.append((time.perf_counter() - started) * 1000)
+    return statistics.median(timings)
+
+
+# --------------------------------------------------------------------- workers
+
+
+def gateway_worker(
+    state: SoakState, base_url: str, secret: str, rng: random.Random, worker_id: int
+) -> None:
+    """Real capability traffic with bursts, steady state and idle gaps."""
+
+    while not state.stop.is_set():
+        phase = rng.choices(["burst", "steady", "idle"], weights=[2, 5, 3])[0]
+        if phase == "idle":
+            state.stop.wait(rng.uniform(8, 25))
+            continue
+        count = rng.randint(3, 6) if phase == "burst" else 1
+        for _ in range(count):
+            if state.stop.is_set():
+                return
+            batch = [rng.choice(SENTENCES) for _ in range(rng.randint(1, 4))]
+            ok, status, latency, body = http(
+                f"{base_url}/api/v1/capabilities/rag.embedding@1/invoke",
+                bearer=secret,
+                method="POST",
+                payload={"input": batch},
+            )
+            execution = body.get("execution") or {}
+            timings = execution.get("timings") or {}
+            state.record(
+                Sample(
+                    at=time.time(),
+                    kind="gateway.rag.embedding",
+                    ok=ok,
+                    status=status,
+                    latency_ms=latency,
+                    detail={
+                        "cold": bool(execution.get("cold")),
+                        "residency": execution.get("residency"),
+                        "queue_ms": timings.get("queue_ms"),
+                        "inference_ms": timings.get("inference_ms"),
+                        "load_ms": timings.get("load_ms"),
+                        "worker": worker_id,
+                        "batch": len(batch),
+                        "code": (body.get("error") or {}).get("code") if not ok else None,
+                    },
+                )
+            )
+            if phase == "burst":
+                state.stop.wait(rng.uniform(0.05, 0.4))
+            else:
+                state.stop.wait(rng.uniform(1.5, 5.0))
+
+
+READ_ROUTES = [
+    ("registry.models", "/api/v1/models", False),
+    ("registry.projects", "/api/v1/projects", False),
+    ("hardware", "/api/v1/hardware", False),
+    ("health.ready", "/api/v1/health/ready", False),
+    ("operations.overview", "/api/v1/admin/operations/overview", True),
+    ("operations.alerts", "/api/v1/admin/operations/alerts?limit=50", True),
+    ("lifecycle.operations", "/api/v1/admin/lifecycle/operations", True),
+    ("migration.plans", "/api/v1/admin/migrations/plans", True),
+    ("recovery.dashboard", "/api/v1/admin/recovery/dashboard", True),
+    ("recovery.backups", "/api/v1/admin/recovery/backups?limit=20", True),
+]
+
+
+def read_worker(state: SoakState, base_url: str, token: str, rng: random.Random) -> None:
+    while not state.stop.is_set():
+        kind, path, needs_token = rng.choice(READ_ROUTES)
+        ok, status, latency, _ = http(
+            f"{base_url}{path}", token=token if needs_token else None, timeout=60.0
+        )
+        state.record(Sample(at=time.time(), kind=kind, ok=ok, status=status, latency_ms=latency))
+        state.stop.wait(rng.uniform(0.5, 3.0))
+
+
+def background_worker(state: SoakState, base_url: str, token: str) -> None:
+    """The operational work the platform performs for itself, driven at a bounded rate."""
+
+    while not state.stop.is_set():
+        for kind, path in (
+            ("ops.capacity_collect", "/api/v1/admin/operations/capacity/collect"),
+            ("ops.slo_evaluate", "/api/v1/admin/operations/slo-evaluations/run"),
+            ("ops.alert_evaluate", "/api/v1/admin/operations/alerts/evaluate"),
+        ):
+            if state.stop.is_set():
+                return
+            ok, status, latency, _ = http(
+                f"{base_url}{path}", token=token, method="POST", timeout=120.0
+            )
+            state.record(
+                Sample(at=time.time(), kind=kind, ok=ok, status=status, latency_ms=latency)
+            )
+        state.stop.wait(60)
+
+
+# --------------------------------------------------------------------- measurement
+
+
+def resource_snapshot(database_url: str, containers: list[str]) -> dict[str, Any]:
+    engine = create_engine(database_url, pool_pre_ping=True)
+    try:
+        with Session(engine) as session:
+            counts = {
+                table: int(
+                    session.execute(text(f"select count(*) from {table}")).scalar_one()  # noqa: S608
+                )
+                for table in (
+                    "audit_events",
+                    "gateway_requests",
+                    "serving_jobs",
+                    "serving_gpu_leases",
+                    "slo_evaluations",
+                    "capacity_snapshots",
+                    "alert_history_events",
+                    "operational_alerts",
+                    "placement_plans",
+                    "hardware_inventory_runs",
+                )
+            }
+            active_leases = int(
+                session.execute(
+                    text(
+                        "select count(*) from serving_gpu_leases "
+                        "where released_at is null and state in "
+                        "('pending','granted','active','held')"
+                    )
+                ).scalar_one()
+            )
+            in_flight = int(
+                session.execute(
+                    text(
+                        "select count(*) from serving_jobs "
+                        "where status in ('queued','leased','running')"
+                    )
+                ).scalar_one()
+            )
+            residency = int(
+                session.execute(
+                    text(
+                        "select count(*) from residency_allocations "
+                        "where state in ('loading','resident','unloading')"
+                    )
+                ).scalar_one()
+            )
+            connections = int(
+                session.execute(
+                    text(
+                        "select count(*) from pg_stat_activity "
+                        "where datname = current_database()"
+                    )
+                ).scalar_one()
+            )
+            gpu = session.execute(
+                text(
+                    "select a.name, t.used_vram_bytes, t.free_vram_bytes, "
+                    "t.gpu_utilization_percent, t.observed_at "
+                    "from accelerator_telemetry_latest t "
+                    "join accelerators a on a.id = t.accelerator_id"
+                )
+            ).all()
+            scheduler = session.execute(
+                text(
+                    "select pressure_state, recovery_candidate, last_observed_at "
+                    "from scheduler_accelerator_states"
+                )
+            ).all()
+            capacity = session.execute(
+                text(
+                    "select gpu_external_bytes, gpu_schedulable_bytes, gpu_managed_resident_bytes, "
+                    "pressure_state, observed_at from capacity_snapshots "
+                    "order by observed_at desc limit 1"
+                )
+            ).first()
+    finally:
+        engine.dispose()
+    return {
+        "observed_at": datetime.now(UTC).isoformat(),
+        "row_counts": counts,
+        "active_leases": active_leases,
+        "serving_jobs_in_flight": in_flight,
+        "residency_allocations": residency,
+        "database_connections": connections,
+        "gpu": [
+            {
+                "name": name,
+                "used_vram_bytes": used,
+                "free_vram_bytes": free,
+                "utilisation_percent": util,
+                "observed_at": observed.isoformat() if observed else None,
+            }
+            for name, used, free, util, observed in gpu
+        ],
+        "scheduler": [
+            {
+                "pressure_state": pressure,
+                "recovery_candidate": bool(candidate),
+                "last_observed_at": observed.isoformat() if observed else None,
+            }
+            for pressure, candidate, observed in scheduler
+        ],
+        "latest_capacity": (
+            {
+                "gpu_external_bytes": capacity[0],
+                "gpu_schedulable_bytes": capacity[1],
+                "gpu_managed_resident_bytes": capacity[2],
+                "pressure_state": capacity[3],
+                "observed_at": capacity[4].isoformat() if capacity[4] else None,
+            }
+            if capacity
+            else None
+        ),
+        "containers": container_stats(containers),
+        "restart_counts": restart_counts(containers),
+    }
+
+
+def percentile(values: list[float], fraction: float) -> float | None:
+    if not values:
+        return None
+    ordered = sorted(values)
+    index = min(len(ordered) - 1, max(0, round(fraction * (len(ordered) - 1))))
+    return round(ordered[index], 3)
+
+
+def summarise(samples: list[Sample]) -> dict[str, Any]:
+    by_kind: dict[str, list[Sample]] = {}
+    for sample in samples:
+        by_kind.setdefault(sample.kind, []).append(sample)
+
+    result: dict[str, Any] = {}
+    for kind, items in sorted(by_kind.items()):
+        latencies = [item.latency_ms for item in items if item.ok]
+        statuses = Counter(item.status for item in items)
+        failures = [item for item in items if not item.ok]
+        capacity = [
+            item
+            for item in failures
+            if item.status in (429, 503) or item.detail.get("code") in ("QUEUE_FULL", "RATE_LIMITED")
+        ]
+        internal = [item for item in failures if item.status >= 500 or item.status == 0]
+        entry: dict[str, Any] = {
+            "requests": len(items),
+            "succeeded": len(items) - len(failures),
+            "capacity_rejected": len(capacity),
+            "internal_errors": len(internal),
+            "statuses": dict(sorted(statuses.items())),
+            "p50_ms": percentile(latencies, 0.50),
+            "p95_ms": percentile(latencies, 0.95),
+            "p99_ms": percentile(latencies, 0.99),
+            "mean_ms": round(statistics.fmean(latencies), 3) if latencies else None,
+            "max_ms": round(max(latencies), 3) if latencies else None,
+        }
+        if kind.startswith("gateway."):
+            cold = [item for item in items if item.ok and item.detail.get("cold")]
+            warm = [item for item in items if item.ok and not item.detail.get("cold")]
+            queue = [
+                float(item.detail["queue_ms"])
+                for item in items
+                if item.ok and item.detail.get("queue_ms") is not None
+            ]
+            inference = [
+                float(item.detail["inference_ms"])
+                for item in items
+                if item.ok and item.detail.get("inference_ms") is not None
+            ]
+            entry["cold_starts"] = len(cold)
+            entry["warm_invocations"] = len(warm)
+            entry["queue_p95_ms"] = percentile(queue, 0.95)
+            entry["inference_p50_ms"] = percentile(inference, 0.50)
+            entry["inference_p95_ms"] = percentile(inference, 0.95)
+            entry["failure_codes"] = dict(
+                Counter(
+                    item.detail.get("code") for item in failures if item.detail.get("code")
+                )
+            )
+        result[kind] = entry
+    return result
+
+
+# --------------------------------------------------------------------- runner
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--base-url", default=DEFAULT_BASE_URL)
+    parser.add_argument("--database-url", default=DEFAULT_DATABASE_URL)
+    parser.add_argument("--token", default=None)
+    parser.add_argument("--secret", default=None, help="gateway client secret")
+    parser.add_argument("--minutes", type=float, default=45.0)
+    parser.add_argument("--gateway-workers", type=int, default=3)
+    parser.add_argument("--read-workers", type=int, default=2)
+    parser.add_argument("--seed", type=int, default=20260827)
+    parser.add_argument("--report", default=None)
+    parser.add_argument("--progress-seconds", type=float, default=120.0)
+    args = parser.parse_args(argv)
+    if not args.database_url:
+        parser.error(
+            "--database-url or MODELFORGE_RUNTIME_DATABASE_URL is required "
+            "(use the non-owner runtime role)"
+        )
+
+    root = Path(__file__).resolve().parents[1]
+    token = args.token
+    if not token and (root / ".env").is_file():
+        for line in (root / ".env").read_text("utf-8").splitlines():
+            if line.startswith("MODELFORGE_OPERATOR_API_KEY="):
+                token = line.split("=", 1)[1].strip()
+    if not token:
+        parser.error("an operator API key is required")
+    if not args.secret:
+        parser.error("a gateway client secret is required")
+
+    containers = [
+        "modelforge-api-1",
+        "modelforge-postgres-1",
+        "modelforge-redis-1",
+        "modelforge-web-1",
+    ]
+
+    floor = connect_floor_ms(args.base_url)
+    print(f"client connect floor: {floor:.1f} ms", flush=True)
+    if floor > MAX_CONNECT_FLOOR_MS:
+        print(
+            f"refusing to soak: the client connect floor is {floor:.1f} ms, above the "
+            f"{MAX_CONNECT_FLOOR_MS:.0f} ms budget. Every latency percentile would be dominated "
+            "by the harness rather than the platform. Use an address that resolves directly "
+            "(127.0.0.1 rather than localhost).",
+            flush=True,
+        )
+        return 2
+
+    print(f"soak: {args.minutes:.1f} minutes, seed {args.seed}", flush=True)
+    before = resource_snapshot(args.database_url, containers)
+    engine = create_engine(args.database_url, pool_pre_ping=True)
+    with Session(engine) as session:
+        baseline = check_invariants(session)
+    engine.dispose()
+    if baseline.violated:
+        print("refusing to soak: invariants are already violated", flush=True)
+        return 2
+
+    state = SoakState()
+    threads: list[threading.Thread] = []
+    for index in range(args.gateway_workers):
+        thread = threading.Thread(
+            target=gateway_worker,
+            args=(state, args.base_url, args.secret, random.Random(args.seed + index), index),
+            daemon=True,
+        )
+        threads.append(thread)
+    for index in range(args.read_workers):
+        thread = threading.Thread(
+            target=read_worker,
+            args=(state, args.base_url, token, random.Random(args.seed + 100 + index)),
+            daemon=True,
+        )
+        threads.append(thread)
+    threads.append(
+        threading.Thread(
+            target=background_worker, args=(state, args.base_url, token), daemon=True
+        )
+    )
+
+    started = time.time()
+    deadline = started + args.minutes * 60
+    for thread in threads:
+        thread.start()
+
+    invariant_checks: list[dict[str, Any]] = []
+    progress: list[dict[str, Any]] = []
+    try:
+        next_progress = started + args.progress_seconds
+        while time.time() < deadline:
+            time.sleep(2)
+            if time.time() >= next_progress:
+                next_progress = time.time() + args.progress_seconds
+                engine = create_engine(args.database_url, pool_pre_ping=True)
+                with Session(engine) as session:
+                    report = check_invariants(session)
+                engine.dispose()
+                with state.lock:
+                    total = len(state.samples)
+                    gateway = sum(
+                        1 for item in state.samples if item.kind.startswith("gateway.")
+                    )
+                    failed = sum(1 for item in state.samples if not item.ok)
+                invariant_checks.append(
+                    {
+                        "at": datetime.now(UTC).isoformat(),
+                        "violated": report.violated,
+                        "violations": [
+                            item.key
+                            for item in report.results
+                            if item.status is InvariantStatus.VIOLATED
+                        ],
+                    }
+                )
+                elapsed = (time.time() - started) / 60
+                entry = {
+                    "minutes": round(elapsed, 1),
+                    "requests": total,
+                    "gateway": gateway,
+                    "failed": failed,
+                    "invariants_violated": report.violated,
+                }
+                progress.append(entry)
+                print(f"  {json.dumps(entry)}", flush=True)
+                if report.violated:
+                    print("  invariant violated; stopping the soak early", flush=True)
+                    break
+    except KeyboardInterrupt:
+        print("  interrupted; reporting partial evidence", flush=True)
+    finally:
+        state.stop.set()
+        for thread in threads:
+            thread.join(timeout=240)
+
+    duration = time.time() - started
+    after = resource_snapshot(args.database_url, containers)
+    engine = create_engine(args.database_url, pool_pre_ping=True)
+    with Session(engine) as session:
+        final = check_invariants(session)
+    engine.dispose()
+
+    with state.lock:
+        samples = list(state.samples)
+
+    drift = {
+        table: after["row_counts"][table] - before["row_counts"][table]
+        for table in before["row_counts"]
+    }
+    report = {
+        "generated_at": datetime.now(UTC).isoformat(),
+        "seed": args.seed,
+        "client_connect_floor_ms": round(floor, 3),
+        "requested_minutes": args.minutes,
+        "actual_duration_seconds": round(duration, 1),
+        "gateway_workers": args.gateway_workers,
+        "read_workers": args.read_workers,
+        "total_requests": len(samples),
+        "total_failures": sum(1 for item in samples if not item.ok),
+        "by_kind": summarise(samples),
+        "resources_before": before,
+        "resources_after": after,
+        "row_growth": drift,
+        "invariant_checks": invariant_checks,
+        "invariants_final": {
+            "checked": final.checked,
+            "violated": final.violated,
+            "violations": [
+                {"key": item.key, "examples": item.violations}
+                for item in final.results
+                if item.status is InvariantStatus.VIOLATED
+            ],
+        },
+        "progress": progress,
+    }
+
+    print(json.dumps({key: report[key] for key in ("actual_duration_seconds", "total_requests", "total_failures")}, indent=2))
+    print(json.dumps(report["by_kind"], indent=2))
+    print("row growth:", json.dumps(drift))
+    print("leases after:", after["active_leases"], "| jobs in flight:", after["serving_jobs_in_flight"])
+    print("invariants:", final.holding, "/", final.checked, "hold")
+
+    if args.report:
+        Path(args.report).write_text(json.dumps(report, indent=2), encoding="utf-8")
+        print(f"report written to {args.report}")
+    return 0 if final.violated == 0 else 1
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/modelforge_scheduled_backup.sh

@@ -0,0 +1,67 @@
+#!/usr/bin/env sh
+# Scheduled ModelForge backup.
+#
+# Creates a backup set, verifies it and applies retention, in that order. Deployment
+# infrastructure owns the schedule — cron, a systemd timer, an Unraid User Script or the
+# `backup-scheduler` service in docker-compose.backup.yml — so the platform never depends on an
+# operator being present to protect its authoritative state.
+#
+# The backup id is derived from the interval so a re-run inside the same interval is a no-op rather
+# than a second copy. A non-zero exit means the platform has no *new* verified recovery point, which
+# is what the BACKUP_STALE and BACKUP_FAILED alerts exist to catch.
+set -eu
+
+BASE_URL="${MODELFORGE_BASE_URL:-http://api:8000}"
+TOKEN="${MODELFORGE_OPERATOR_API_KEY:?operator API key is required for scheduled backups}"
+PREFIX="${MODELFORGE_BACKUP_ID_PREFIX:-scheduled}"
+STAMP="$(date -u +%Y%m%d-%H%M)"
+BACKUP_ID="${PREFIX}-${STAMP}"
+REASON="${MODELFORGE_BACKUP_REASON:-Scheduled ModelForge control-plane backup}"
+
+log() { printf '%s modelforge-backup %s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$*"; }
+
+# Extract the first occurrence of a top-level JSON string field. Deliberately not greedy: the
+# response embeds nested objects and a greedy match would read the wrong one.
+field() {
+  printf '%s' "$2" | grep -o "\"$1\":\"[^\"]*\"" | head -n 1 | cut -d'"' -f4
+}
+
+api() {
+  method="$1"
+  path="$2"
+  shift 2
+  curl -sS -X "${method}" -H "X-ModelForge-Admin-Token: ${TOKEN}" \
+    -H "Content-Type: application/json" "${BASE_URL}${path}" "$@"
+}
+
+log "creating ${BACKUP_ID}"
+created="$(api POST /api/v1/admin/recovery/backups \
+  --data "{\"backup_id\":\"${BACKUP_ID}\",\"reason\":\"${REASON}\"}")"
+
+backup_uuid="$(field id "${created}")"
+if [ -z "${backup_uuid}" ]; then
+  log "backup creation did not return an id: ${created}"
+  exit 1
+fi
+
+state="$(field state "${created}")"
+log "created ${BACKUP_ID} (${backup_uuid}) state=${state}"
+if [ "${state}" = "FAILED" ]; then
+  log "backup failed; leaving it journalled as evidence"
+  exit 1
+fi
+
+log "verifying ${BACKUP_ID}"
+verified="$(api POST "/api/v1/admin/recovery/backups/${backup_uuid}/verify")"
+verified_state="$(field state "${verified}")"
+log "verification result state=${verified_state}"
+if [ "${verified_state}" != "VERIFIED" ]; then
+  log "backup is not restore eligible; see RUNBOOK_BACKUP_FAILURE.md"
+  exit 1
+fi
+
+# Retention runs only after a new verified backup exists, so the last known-good recovery point can
+# never be expired in favour of one that has not proven itself.
+log "applying retention"
+api POST /api/v1/admin/recovery/retention/run > /dev/null
+log "scheduled backup complete: ${BACKUP_ID} VERIFIED"

scripts/preflight.py

@@ -0,0 +1,282 @@
+"""Check a host before installing ModelForge, and say exactly what is wrong.
+
+Run this first on a clean machine. It answers the question an operator actually has — "will this
+work here?" — before anything is downloaded, built or started, and it never changes the system it
+is inspecting.
+
+    python scripts/preflight.py
+    python scripts/preflight.py --production   # also check the production configuration rules
+
+Exit status is 0 when the host can run ModelForge, 1 when something must be fixed.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import shutil
+import socket
+import subprocess  # noqa: S404 - fixed argv, never a shell
+import sys
+from dataclasses import dataclass
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[1]
+sys.path.insert(0, str(ROOT / "backend" / "src"))
+
+from modelforge_api.domain.release import (  # noqa: E402
+    MINIMUM_POSTGRES_MAJOR,
+    PRODUCT_NAME,
+    PRODUCT_VERSION,
+)
+
+MINIMUM_DOCKER_MAJOR = 24
+MINIMUM_COMPOSE_MAJOR = 2
+RECOMMENDED_FREE_BYTES = 50 * 1024**3
+
+#: Ports the deployment publishes, and the variable that moves each one.
+PUBLISHED_PORTS = {
+    "MODELFORGE_API_PUBLISHED_PORT": 8000,
+    "MODELFORGE_WEB_PORT": 3000,
+    "MODELFORGE_POSTGRES_PORT": 5432,
+    "MODELFORGE_REDIS_PORT": 6379,
+}
+
+
+@dataclass
+class Check:
+    name: str
+    ok: bool
+    detail: str
+    fatal: bool = True
+
+
+def run(*args: str) -> tuple[int, str]:
+    try:
+        completed = subprocess.run(  # noqa: S603 - fixed argv, no shell
+            list(args),
+            capture_output=True,
+            text=True,
+            encoding="utf-8",
+            errors="replace",
+            timeout=60,
+            check=False,
+        )
+    except (OSError, subprocess.TimeoutExpired) as error:
+        return 1, f"{type(error).__name__}"
+    return completed.returncode, (completed.stdout or completed.stderr or "").strip()
+
+
+def _major(value: str) -> int:
+    digits = ""
+    for character in value.lstrip("v"):
+        if character.isdigit():
+            digits += character
+        else:
+            break
+    return int(digits) if digits else 0
+
+
+def check_docker() -> list[Check]:
+    checks: list[Check] = []
+    if shutil.which("docker") is None:
+        return [Check("docker", False, "docker is not on PATH")]
+    code, version = run("docker", "version", "--format", "{{.Server.Version}}")
+    if code != 0:
+        return [Check("docker", False, f"the Docker daemon is not reachable: {version[:120]}")]
+    major = _major(version)
+    checks.append(
+        Check(
+            "docker engine",
+            major >= MINIMUM_DOCKER_MAJOR,
+            f"{version} (minimum {MINIMUM_DOCKER_MAJOR})",
+        )
+    )
+    code, compose = run("docker", "compose", "version", "--short")
+    checks.append(
+        Check(
+            "docker compose",
+            code == 0 and _major(compose) >= MINIMUM_COMPOSE_MAJOR,
+            f"{compose or 'absent'} (minimum {MINIMUM_COMPOSE_MAJOR}.x)",
+        )
+    )
+    return checks
+
+
+def check_gpu() -> list[Check]:
+    """A GPU is required on a compute node, not on the control-plane host."""
+
+    if shutil.which("nvidia-smi") is None:
+        return [
+            Check(
+                "nvidia driver",
+                True,
+                "nvidia-smi is absent; required only on a GPU compute node",
+                fatal=False,
+            )
+        ]
+    code, output = run("nvidia-smi", "--query-gpu=name,driver_version", "--format=csv,noheader")
+    if code != 0:
+        return [Check("nvidia driver", False, "nvidia-smi failed", fatal=False)]
+    return [Check("nvidia driver", True, output.replace("\n", "; "), fatal=False)]
+
+
+def check_ports() -> list[Check]:
+    checks = []
+    for variable, default in PUBLISHED_PORTS.items():
+        port = int(os.environ.get(variable, default))
+        connection = socket.socket()
+        connection.settimeout(1)
+        try:
+            connection.connect(("127.0.0.1", port))
+            occupied = True
+        except OSError:
+            occupied = False
+        finally:
+            connection.close()
+        checks.append(
+            Check(
+                f"port {port}",
+                not occupied,
+                f"already in use; move it with {variable}" if occupied else "free",
+                fatal=False,
+            )
+        )
+    return checks
+
+
+def check_storage(path: Path) -> list[Check]:
+    try:
+        usage = shutil.disk_usage(path)
+    except OSError as error:
+        return [Check("storage", False, f"{path} is not readable: {error}")]
+    free_gib = usage.free / 1024**3
+    return [
+        Check(
+            "free storage",
+            usage.free >= RECOMMENDED_FREE_BYTES,
+            f"{free_gib:.1f} GiB free at {path} "
+            f"(recommended {RECOMMENDED_FREE_BYTES / 1024**3:.0f} GiB for model artifacts)",
+            fatal=False,
+        )
+    ]
+
+
+def check_configuration(production: bool) -> list[Check]:
+    env_file = ROOT / ".env"
+    example = ROOT / ".env.example"
+    checks = [
+        Check(".env.example", example.is_file(), "present" if example.is_file() else "missing")
+    ]
+    if not env_file.is_file():
+        checks.append(
+            Check(
+                ".env",
+                not production,
+                "absent; copy .env.example to .env and fill in the generated secrets",
+                fatal=production,
+            )
+        )
+        return checks
+    checks.append(Check(".env", True, "present"))
+    if production:
+        from modelforge_api.services.startup_validation import (
+            StartupFailureCode,
+            validate_settings,
+        )
+        from modelforge_api.settings import Settings
+
+        settings = Settings(_env_file=str(env_file))  # type: ignore[call-arg]
+        report = validate_settings(settings)
+        # Storage roots are container paths. Validating them against the host is meaningless — the
+        # host has no /data/artifacts and is not supposed to — so they are checked inside the
+        # container at startup, which is the only place the answer means anything.
+        relevant = [
+            problem
+            for problem in report.problems
+            if problem.code is not StartupFailureCode.INVALID_STORAGE_ROOT
+        ]
+        for problem in relevant:
+            checks.append(Check(problem.setting, False, f"{problem.code}: {problem.message}"))
+        if not relevant:
+            checks.append(
+                Check(
+                    "production configuration",
+                    True,
+                    "no problems found; storage roots are validated inside the container",
+                )
+            )
+    return checks
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--production", action="store_true")
+    parser.add_argument("--storage-path", default=str(ROOT))
+    parser.add_argument("--json", action="store_true")
+    args = parser.parse_args(argv)
+
+    checks: list[Check] = []
+    checks.append(
+        Check(
+            "python",
+            sys.version_info >= (3, 12),
+            f"{sys.version.split()[0]} (minimum 3.12)",
+            fatal=False,
+        )
+    )
+    checks += check_docker()
+    checks += check_gpu()
+    checks += check_ports()
+    checks += check_storage(Path(args.storage_path))
+    checks += check_configuration(args.production)
+    checks.append(
+        Check(
+            "postgresql",
+            True,
+            f"provided by the compose deployment; minimum major {MINIMUM_POSTGRES_MAJOR}",
+            fatal=False,
+        )
+    )
+
+    blocking = [check for check in checks if not check.ok and check.fatal]
+    advisory = [check for check in checks if not check.ok and not check.fatal]
+
+    if args.json:
+        print(
+            json.dumps(
+                {
+                    "product": PRODUCT_NAME,
+                    "version": PRODUCT_VERSION,
+                    "ready": not blocking,
+                    "checks": [
+                        {
+                            "name": check.name,
+                            "ok": check.ok,
+                            "detail": check.detail,
+                            "fatal": check.fatal,
+                        }
+                        for check in checks
+                    ],
+                },
+                indent=2,
+            )
+        )
+    else:
+        print(f"{PRODUCT_NAME} {PRODUCT_VERSION} — host preflight")
+        for check in checks:
+            mark = "OK  " if check.ok else ("FAIL" if check.fatal else "WARN")
+            print(f"  {mark}  {check.name:24} {check.detail}")
+        print()
+        if blocking:
+            print(f"{len(blocking)} problem(s) must be fixed before installing.")
+        elif advisory:
+            print(f"Host is ready. {len(advisory)} advisory note(s) above.")
+        else:
+            print("Host is ready.")
+    return 1 if blocking else 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/rc_server_acceptance.py

@@ -0,0 +1,442 @@
+"""Build, scan and clean-install a ModelForge candidate in an isolated Compose project."""
+
+from __future__ import annotations
+
+import argparse
+import atexit
+import base64
+import json
+import os
+import secrets
+import shutil
+import subprocess
+import sys
+import time
+from datetime import UTC, datetime
+from pathlib import Path
+from urllib.parse import urlparse
+
+
+ROOT = Path(__file__).resolve().parents[1]
+IMAGES = (
+    "modelforge-api",
+    "modelforge-web",
+    "modelforge-node-agent",
+    "modelforge-runtime-worker",
+)
+CONFIG_SEED_IMAGE = (
+    "redis:7-alpine@sha256:ff02b58f971e7d7d156a1267e283fcbbeee91773b6aa36c49dac28ecfe28eadf"
+)
+
+
+def run(
+    *args: str,
+    cwd: Path = ROOT,
+    env: dict[str, str] | None = None,
+    check: bool = True,
+    capture: bool = False,
+) -> str:
+    completed = subprocess.run(
+        list(args),
+        cwd=cwd,
+        env=env,
+        text=True,
+        encoding="utf-8",
+        errors="replace",
+        stdout=subprocess.PIPE if capture else None,
+        stderr=subprocess.STDOUT if capture else None,
+        check=False,
+    )
+    output = (completed.stdout or "").strip()
+    if check and completed.returncode:
+        raise RuntimeError(f"{' '.join(args)} failed ({completed.returncode}): {output[-2000:]}")
+    return output
+
+
+def container_http_status(
+    container: str, url: str, *, operator: bool = False
+) -> tuple[int, str]:
+    probe = """
+import json
+import os
+import sys
+import urllib.error
+import urllib.request
+
+headers = {}
+if sys.argv[2] == "operator":
+    headers["X-ModelForge-Admin-Token"] = os.environ["MODELFORGE_OPERATOR_API_KEY"]
+request = urllib.request.Request(sys.argv[1], headers=headers)
+try:
+    with urllib.request.urlopen(request, timeout=10) as response:
+        status = response.status
+        body = response.read().decode("utf-8", "replace")
+except urllib.error.HTTPError as error:
+    status = error.code
+    body = error.read().decode("utf-8", "replace")
+except OSError as error:
+    status = 0
+    body = str(error)
+print(json.dumps({"status": status, "body": body}))
+"""
+    output = run(
+        "docker",
+        "exec",
+        container,
+        "python",
+        "-c",
+        probe,
+        url,
+        "operator" if operator else "anonymous",
+        capture=True,
+    )
+    result = json.loads(output)
+    return int(result["status"]), str(result["body"])
+
+
+def wait_for_container_status(
+    container: str, url: str, expected: int, timeout: int = 120
+) -> str:
+    deadline = time.monotonic() + timeout
+    last = "no response"
+    while time.monotonic() < deadline:
+        try:
+            status, body = container_http_status(container, url)
+            last = f"HTTP {status}: {body[:200]}"
+            if status == expected:
+                return body
+        except OSError as exc:
+            last = str(exc)
+        time.sleep(2)
+    raise RuntimeError(f"{url} did not reach HTTP {expected}: {last}")
+
+
+def provision_database_roles(
+    compose: tuple[str, ...], env: dict[str, str], database: str
+) -> None:
+    """Apply the idempotent role bootstrap through the Docker API.
+
+    Gitea Actions talks to a sibling Docker daemon. Host bind mounts therefore resolve on the
+    daemon, not in the job container, so the normal init-directory mount is intentionally not
+    relied on by acceptance. ``docker cp`` preserves the exact release SQL while working for both
+    local and remote daemons.
+    """
+
+    postgres = run(*compose, "ps", "--quiet", "postgres", env=env, capture=True)
+    if not postgres:
+        raise RuntimeError("acceptance PostgreSQL container was not created")
+    bootstrap = ROOT / "deploy" / "postgres" / "init" / "001-modelforge-roles.sql"
+    container_path = "/tmp/001-modelforge-roles.sql"
+    run("docker", "cp", str(bootstrap), f"{postgres}:{container_path}")
+    try:
+        run(
+            "docker",
+            "exec",
+            postgres,
+            "psql",
+            "--username",
+            env.get("MODELFORGE_POSTGRES_ADMIN_USER", "postgres"),
+            "--dbname",
+            database,
+            "--file",
+            container_path,
+        )
+    finally:
+        run("docker", "exec", postgres, "rm", "--force", container_path, check=False)
+
+
+def provision_config_volume(project: str) -> None:
+    """Populate the read-only API config volume through the Docker API."""
+
+    volume = f"{project}_acceptance-config"
+    seed = f"{project}-config-seed"
+    run(
+        "docker",
+        "volume",
+        "create",
+        "--label",
+        f"com.docker.compose.project={project}",
+        "--label",
+        "com.docker.compose.volume=acceptance-config",
+        volume,
+    )
+    run(
+        "docker",
+        "create",
+        "--name",
+        seed,
+        "--label",
+        f"com.docker.compose.project={project}",
+        "--volume",
+        f"{volume}:/app/config",
+        CONFIG_SEED_IMAGE,
+        "true",
+    )
+    try:
+        run("docker", "cp", f"{ROOT / 'config'}/.", f"{seed}:/app/config")
+    finally:
+        run("docker", "container", "rm", "--force", seed, check=False)
+
+
+def cleanup_candidate_images(version: str, commit: str) -> None:
+    """Remove only tags stamped by this exact acceptance commit."""
+
+    for image in IMAGES:
+        tag = f"{image}:{version}"
+        revision = run(
+            "docker",
+            "inspect",
+            "--format",
+            '{{index .Config.Labels "org.opencontainers.image.revision"}}',
+            tag,
+            check=False,
+            capture=True,
+        )
+        if revision == commit:
+            print(f"Removing acceptance image {tag}", flush=True)
+            run("docker", "image", "rm", "--force", tag, check=False)
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--public-api-origin", required=True)
+    parser.add_argument("--output", type=Path, default=Path("acceptance-evidence"))
+    parser.add_argument("--trivy", default="trivy")
+    parser.add_argument("--project-suffix", default=os.environ.get("GITHUB_RUN_ID", "manual"))
+    args = parser.parse_args()
+
+    origin = args.public_api_origin.rstrip("/")
+    parsed = urlparse(origin)
+    if parsed.scheme not in {"http", "https"} or not parsed.netloc or parsed.path:
+        raise SystemExit("--public-api-origin must be a bare absolute HTTP(S) origin")
+    if shutil.which("docker") is None:
+        raise SystemExit("Docker is required on the acceptance runner")
+    if shutil.which(args.trivy) is None:
+        raise SystemExit(f"Trivy executable not found: {args.trivy}")
+
+    commit = run("git", "rev-parse", "HEAD", capture=True)
+    if run("git", "status", "--porcelain", capture=True):
+        raise SystemExit("Acceptance must run from a clean, exact source commit")
+    version = (ROOT / "VERSION").read_text(encoding="utf-8").strip()
+    atexit.register(cleanup_candidate_images, version, commit)
+    built_at = datetime.now(UTC).isoformat()
+    output = args.output.resolve()
+    output.mkdir(parents=True, exist_ok=True)
+
+    project_suffix = "".join(ch for ch in args.project_suffix.lower() if ch.isalnum())[-24:]
+    project = f"modelforge-rc-{project_suffix or secrets.token_hex(6)}"
+    if not project.startswith("modelforge-rc-"):
+        raise SystemExit("refusing a non-RC Compose project name")
+
+    run(
+        sys.executable,
+        str(ROOT / "scripts" / "release_build.py"),
+        "--output",
+        str(output / "release"),
+        "--public-api-origin",
+        origin,
+    )
+
+    image_records: list[dict[str, str]] = []
+    for image in IMAGES:
+        tag = f"{image}:{version}"
+        image_id = run("docker", "inspect", "--format", "{{.Id}}", tag, capture=True)
+        raw_report = output / f"trivy-{image}.json"
+        trivy_summary = output / f"trivy-{image}-summary.json"
+        run(
+            args.trivy,
+            "image",
+            "--scanners",
+            "vuln",
+            "--severity",
+            "UNKNOWN,LOW,MEDIUM,HIGH,CRITICAL",
+            "--format",
+            "json",
+            "--output",
+            str(raw_report),
+            image_id,
+        )
+        run(
+            sys.executable,
+            str(ROOT / "scripts" / "validate_trivy_report.py"),
+            "--report",
+            str(raw_report),
+            "--image-id",
+            image_id,
+            "--summary",
+            str(trivy_summary),
+            "--reviewed-unfixed",
+            str(ROOT / "config" / "public-candidate-unfixed-vulnerabilities.json"),
+        )
+        image_records.append({"name": image, "tag": tag, "image_id": image_id})
+
+    admin_password = secrets.token_hex(24)
+    owner_password = secrets.token_hex(24)
+    runtime_password = secrets.token_hex(24)
+    operator_key = secrets.token_urlsafe(48)
+    backup_key = base64.urlsafe_b64encode(secrets.token_bytes(32)).decode("ascii")
+    database = "modelforge_rc"
+    env = os.environ.copy()
+    env.update(
+        {
+            "COMPOSE_PROJECT_NAME": project,
+            "MODELFORGE_VERSION": version,
+            "MODELFORGE_COMMIT": commit,
+            "MODELFORGE_BUILT_AT": built_at,
+            "MODELFORGE_API_IMAGE": f"modelforge-api:{version}",
+            "MODELFORGE_WEB_IMAGE": f"modelforge-web:{version}",
+            "MODELFORGE_NODE_AGENT_IMAGE": f"modelforge-node-agent:{version}",
+            "MODELFORGE_RUNTIME_WORKER_IMAGE": f"modelforge-runtime-worker:{version}",
+            "MODELFORGE_POSTGRES_DB": database,
+            "MODELFORGE_POSTGRES_ADMIN_PASSWORD": admin_password,
+            "MODELFORGE_MIGRATION_DB_PASSWORD": owner_password,
+            "MODELFORGE_RUNTIME_DB_PASSWORD": runtime_password,
+            "MODELFORGE_MIGRATION_DATABASE_URL": (
+                f"postgresql+psycopg://modelforge:{owner_password}@postgres:5432/{database}"
+            ),
+            "MODELFORGE_RUNTIME_DATABASE_URL": (
+                f"postgresql+psycopg://modelforge_runtime:{runtime_password}@postgres:5432/{database}"
+            ),
+            "MODELFORGE_OPERATOR_API_KEY": operator_key,
+            "MODELFORGE_BACKUP_ENCRYPTION_KEY": backup_key,
+            "MODELFORGE_CORS_ORIGINS": origin,
+            "VITE_API_BASE_URL": origin,
+            "MODELFORGE_SOURCE_COMMIT": commit,
+            "MODELFORGE_SOURCE_REFERENCE": "rc-acceptance",
+            "MODELFORGE_SOURCE_REPOSITORY": "public-source-candidate",
+            "MODELFORGE_API_BIND": "127.0.0.1",
+            "MODELFORGE_API_PUBLISHED_PORT": "0",
+            "MODELFORGE_WEB_BIND": "127.0.0.1",
+            "MODELFORGE_WEB_PORT": "0",
+            "MODELFORGE_POSTGRES_BIND": "127.0.0.1",
+            "MODELFORGE_POSTGRES_PORT": "0",
+            "MODELFORGE_REDIS_BIND": "127.0.0.1",
+            "MODELFORGE_REDIS_PORT": "0",
+            "MODELFORGE_ALLOW_REMOTE_CODE": "false",
+            "MODELFORGE_RESTORE_ALLOW_PRODUCTION_TARGET": "false",
+        }
+    )
+    acceptance_override = output / "acceptance-compose.override.yml"
+    acceptance_override.write_text(
+        "services:\n"
+        "  api:\n"
+        "    volumes:\n"
+        "      - type: volume\n"
+        "        source: acceptance-config\n"
+        "        target: /app/config\n"
+        "        read_only: true\n"
+        "volumes:\n"
+        "  acceptance-config:\n",
+        encoding="utf-8",
+        newline="\n",
+    )
+    compose = (
+        "docker",
+        "compose",
+        "-p",
+        project,
+        "-f",
+        str(ROOT / "docker-compose.yml"),
+        "-f",
+        str(ROOT / "docker-compose.production.yml"),
+        "-f",
+        str(acceptance_override),
+    )
+
+    summary: dict[str, object] = {
+        "schema_version": 1,
+        "project": project,
+        "source_commit": commit,
+        "version": version,
+        "public_api_origin": origin,
+        "production_changed": False,
+        "compute_identity_created": False,
+        "images": image_records,
+        "checks": {},
+    }
+    try:
+        run(*compose, "config", "-q", env=env)
+        run(*compose, "up", "-d", "--no-build", "--wait", "postgres", "redis", env=env)
+        provision_database_roles(compose, env, database)
+        provision_config_volume(project)
+        run(*compose, "up", "-d", "--no-build", "--wait", "api", "web", env=env)
+        api_container = run(*compose, "ps", "--quiet", "api", env=env, capture=True)
+        if not api_container:
+            raise RuntimeError("acceptance API container was not created")
+        api = "http://127.0.0.1:8000"
+
+        wait_for_container_status(api_container, f"{api}/api/v1/health/live", 200)
+        wait_for_container_status(api_container, f"{api}/api/v1/health/ready", 200)
+        version_status, version_body = container_http_status(
+            api_container, f"{api}/api/v1/version"
+        )
+        unauthenticated, _ = container_http_status(
+            api_container, f"{api}/api/v1/admin/recovery/dashboard"
+        )
+        authenticated, _ = container_http_status(
+            api_container,
+            f"{api}/api/v1/admin/recovery/dashboard",
+            operator=True,
+        )
+        wait_for_container_status(api_container, "http://web:3000/", 200)
+        if version_status != 200 or unauthenticated != 401 or authenticated != 200:
+            raise RuntimeError(
+                "acceptance boundary mismatch: "
+                f"version={version_status}, unauthenticated={unauthenticated}, "
+                f"authenticated={authenticated}"
+            )
+        version_payload = json.loads(version_body)
+        if version_payload.get("version") != version:
+            raise RuntimeError(f"running version does not match {version}: {version_payload}")
+
+        volumes = run(
+            "docker",
+            "volume",
+            "ls",
+            "--filter",
+            f"label=com.docker.compose.project={project}",
+            "--format",
+            "{{.Name}}",
+            capture=True,
+        ).splitlines()
+        if not volumes or any(not volume.startswith(f"{project}_") for volume in volumes):
+            raise RuntimeError(f"Compose volumes are not isolated under {project}: {volumes}")
+        summary["checks"] = {
+            "live": 200,
+            "ready": 200,
+            "version": version_status,
+            "admin_without_key": unauthenticated,
+            "admin_with_key": authenticated,
+            "console": 200,
+            "database_roles_provisioned": True,
+            "network_probe_container": "api",
+            "isolated_volumes": sorted(volumes),
+        }
+        summary["result"] = "PASS"
+        (output / "compose-ps.json").write_text(
+            run(*compose, "ps", "--format", "json", env=env, capture=True) + "\n",
+            encoding="utf-8",
+            newline="\n",
+        )
+    except Exception:
+        summary["result"] = "FAIL"
+        (output / "compose-logs.txt").write_text(
+            run(*compose, "logs", "--no-color", env=env, check=False, capture=True) + "\n",
+            encoding="utf-8",
+            newline="\n",
+        )
+        raise
+    finally:
+        run(*compose, "down", "--volumes", "--remove-orphans", env=env, check=False)
+        (output / "acceptance-summary.json").write_text(
+            json.dumps(summary, indent=2, sort_keys=True) + "\n",
+            encoding="utf-8",
+            newline="\n",
+        )
+
+    print(f"Isolated RC acceptance PASS for {commit} in {project}")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/release_build.py

@@ -0,0 +1,466 @@
+"""Build a ModelForge release: validate, build, inventory, sign the inventory, package.
+
+One command, so that what ships is what was tested rather than whatever happened to be in the
+working tree. Everything it emits is derived from the repository and the built images — nothing is
+typed in by hand, because a release manifest an operator cannot verify is decoration.
+
+    python scripts/release_build.py --output dist
+
+The build refuses to run against a dirty working tree unless told otherwise. A release built from
+uncommitted changes cannot be reproduced, and its recorded source commit would be a lie.
+"""
+
+from __future__ import annotations
+
+import argparse
+import gzip
+import hashlib
+import json
+import os
+import shutil
+import subprocess
+import sys
+import tarfile
+import tempfile
+from datetime import UTC, datetime
+from pathlib import Path
+from typing import Any
+from urllib.parse import urlparse
+
+ROOT = Path(__file__).resolve().parents[1]
+sys.path.insert(0, str(ROOT / "backend" / "src"))
+
+from modelforge_api.domain.release import (
+    CURRENT_AGENT_PROTOCOL_VERSION,
+    MINIMUM_POSTGRES_MAJOR,
+    MINIMUM_UPGRADE_SOURCE,
+    PRODUCT_NAME,
+    PRODUCT_VERSION,
+    RELEASE_CHANNEL,
+    SUPPORTED_AGENT_PROTOCOL_VERSIONS,
+    SUPPORTED_SCHEMA_REVISIONS,
+    SUPPORTED_UPGRADE_SOURCE_SCHEMA_REVISIONS,
+    TARGET_SCHEMA_REVISION,
+)
+
+#: image name -> (build context, Dockerfile path), both relative to the repository root. The Node
+#: Agent and Runtime Worker build from the repository root because their Dockerfiles copy a
+#: component directory from that shared context. Keeping the four distributable images in one
+#: inventory makes it impossible for the private runtime boundary to disappear from a release.
+IMAGES = {
+    "modelforge-api": ("backend", "backend/Dockerfile"),
+    "modelforge-web": ("frontend", "frontend/Dockerfile"),
+    "modelforge-node-agent": (".", "node-agent/Dockerfile"),
+    "modelforge-runtime-worker": (".", "runtime-worker/Dockerfile"),
+}
+
+#: What a release tarball contains. Deliberately no model weights, no .env, no database dump and no
+#: credential of any kind — see the packaging test, which fails if that changes.
+ARTIFACT_PATHS = (
+    "docker-compose.yml",
+    "docker-compose.production.yml",
+    "docker-compose.node-agent.yml",
+    "docker-compose.runtime-worker.yml",
+    "docker-compose.gpu.yml",
+    "docker-compose.backup.yml",
+    "docker-compose.dr.yml",
+    ".env.example",
+    "VERSION",
+    "README.md",
+    "config",
+    "docs",
+    "scripts/bootstrap.py",
+    "scripts/preflight.py",
+)
+
+
+def run(*args: str, check: bool = True, cwd: Path | None = None) -> str:
+    completed = subprocess.run(
+        list(args),
+        capture_output=True,
+        text=True,
+        encoding="utf-8",
+        errors="replace",
+        cwd=cwd or ROOT,
+        check=False,
+    )
+    if check and completed.returncode != 0:
+        raise RuntimeError(f"{' '.join(args)} failed: {completed.stderr.strip()[:500]}")
+    return (completed.stdout or "").strip()
+
+
+def sha256_file(path: Path) -> str:
+    digest = hashlib.sha256()
+    with path.open("rb") as handle:
+        for chunk in iter(lambda: handle.read(1024 * 1024), b""):
+            digest.update(chunk)
+    return digest.hexdigest()
+
+
+def source_identity(allow_dirty: bool) -> dict[str, Any]:
+    commit = run("git", "rev-parse", "HEAD")
+    dirty = bool(run("git", "status", "--porcelain"))
+    if dirty and not allow_dirty:
+        raise SystemExit(
+            "refusing to build a release from a dirty working tree: the recorded source commit "
+            "would not describe what was built. Commit first, or pass --allow-dirty for a "
+            "rehearsal build."
+        )
+    describe = run("git", "describe", "--tags", "--always", check=False)
+    return {
+        "commit": commit,
+        "describe": describe or None,
+        "dirty": dirty,
+        "repository": run("git", "remote", "get-url", "origin", check=False) or None,
+        "branch": run("git", "rev-parse", "--abbrev-ref", "HEAD", check=False) or None,
+    }
+
+
+def source_date_epoch(built_at: str) -> int:
+    """Translate the declared build time into BuildKit's reproducible timestamp contract."""
+
+    parsed = datetime.fromisoformat(built_at.replace("Z", "+00:00"))
+    if parsed.tzinfo is None:
+        raise ValueError("build timestamp must include a UTC offset")
+    return int(parsed.timestamp())
+
+
+def manifest_digest(repo_digest: str | None) -> str | None:
+    """Return the pullable OCI digest rather than Docker's local config/image identifier."""
+
+    if not repo_digest or "@" not in repo_digest:
+        return None
+    digest = repo_digest.rsplit("@", 1)[1]
+    return digest if digest.startswith("sha256:") else None
+
+
+#: Images whose behaviour depends on the origin the console is served against. Vite inlines
+#: VITE_* variables into the bundle at build time, so this is a property of the artifact rather
+#: than of the deployment that runs it.
+ORIGIN_DEPENDENT_IMAGES = frozenset({"modelforge-web"})
+
+
+def normalise_public_api_origin(value: str) -> str:
+    """Validate the console's compiled-in API origin, or refuse to build a release without one.
+
+    v1.2.0 shipped a console that could not reach its own API. The release build never passed
+    ``VITE_API_BASE_URL``, so Vite compiled the Dockerfile's development default into an immutable
+    bundle, and the nginx CSP — derived from the same argument — hardcoded the same wrong origin.
+    A convenience default is exactly right for ``docker compose up`` on a laptop and exactly wrong
+    for a release artifact, so the release path refuses to guess.
+    """
+
+    origin = value.strip().rstrip("/")
+    if not origin:
+        raise SystemExit(
+            "a release build needs the public API origin the console will be served against. "
+            "Pass --public-api-origin, or set MODELFORGE_PUBLIC_API_ORIGIN. It is compiled into "
+            "the bundle and into the CSP, so it cannot be corrected after the fact by the "
+            "deployment that runs the image."
+        )
+    parsed = urlparse(origin)
+    if parsed.scheme not in {"http", "https"} or not parsed.netloc:
+        raise SystemExit(
+            f"public API origin must be an absolute http(s) origin, got {value!r}. "
+            "Example: https://modelforge.example.com or http://192.0.2.10:18000"
+        )
+    if parsed.path or parsed.query or parsed.fragment:
+        raise SystemExit(
+            f"public API origin must be a bare origin with no path, query or fragment, "
+            f"got {value!r}. The console appends its own /api/v1 paths."
+        )
+    return origin
+
+
+def build_images(
+    version: str, commit: str, built_at: str, public_api_origin: str
+) -> dict[str, dict[str, Any]]:
+    built: dict[str, dict[str, Any]] = {}
+    epoch = source_date_epoch(built_at)
+    for image, (context, dockerfile) in IMAGES.items():
+        tag = f"{image}:{version}"
+        print(f"  building {tag}", flush=True)
+        origin_args: tuple[str, ...] = ()
+        if image in ORIGIN_DEPENDENT_IMAGES:
+            origin_args = ("--build-arg", f"VITE_API_BASE_URL={public_api_origin}")
+            print(f"    public API origin {public_api_origin}", flush=True)
+        with tempfile.TemporaryDirectory(prefix="modelforge-build-") as temporary:
+            metadata_path = Path(temporary) / "metadata.json"
+            run(
+                "docker",
+                "build",
+                # BuildKit attestations carry an exporter invocation timestamp and make the local
+                # manifest-list ID vary. ModelForge emits its own commit-bound CycloneDX and image
+                # provenance below, so suppress the duplicate nondeterministic attestations.
+                "--provenance=false",
+                "--sbom=false",
+                "--metadata-file",
+                str(metadata_path),
+                "-t",
+                tag,
+                "-f",
+                str(ROOT / dockerfile),
+                "--build-arg",
+                f"MODELFORGE_VERSION={version}",
+                "--build-arg",
+                f"MODELFORGE_COMMIT={commit}",
+                "--build-arg",
+                f"MODELFORGE_BUILT_AT={built_at}",
+                "--build-arg",
+                f"SOURCE_DATE_EPOCH={epoch}",
+                *origin_args,
+                str(ROOT / context),
+            )
+            build_metadata = json.loads(metadata_path.read_text(encoding="utf-8"))
+        image_id = run("docker", "inspect", "--format", "{{.Id}}", tag)
+        architecture = run("docker", "inspect", "--format", "{{.Architecture}}", tag)
+        labels = json.loads(
+            run("docker", "inspect", "--format", "{{json .Config.Labels}}", tag) or "{}"
+        )
+        repo_digest = (
+            run("docker", "inspect", "--format", "{{index .RepoDigests 0}}", tag, check=False)
+            or None
+        )
+        built[image] = {
+            "tag": tag,
+            "image_id": image_id,
+            "image_digest": manifest_digest(repo_digest)
+            or build_metadata.get("containerimage.digest"),
+            "repo_digest": repo_digest,
+            "architecture": architecture,
+            "labels": {
+                key: value
+                for key, value in (labels or {}).items()
+                if key.startswith("org.opencontainers.image.")
+            },
+        }
+    return built
+
+
+def package(output: Path, version: str, sbom_source: Path | None = None) -> Path:
+    """Assemble the release tarball from tracked paths only."""
+
+    staging = output / f"modelforge-{version}"
+    if staging.exists():
+        shutil.rmtree(staging)
+    staging.mkdir(parents=True)
+    for relative in ARTIFACT_PATHS:
+        source = ROOT / relative
+        if not source.exists():
+            print(f"  note: {relative} is absent and was not packaged", flush=True)
+            continue
+        target = staging / relative
+        target.parent.mkdir(parents=True, exist_ok=True)
+        if source.is_dir():
+            shutil.copytree(source, target, ignore=shutil.ignore_patterns("__pycache__", "*.pyc"))
+        else:
+            shutil.copy2(source, target)
+
+    if sbom_source is not None:
+        sbom_target = staging / "docs" / "security" / "sbom"
+        if sbom_target.exists():
+            shutil.rmtree(sbom_target)
+        shutil.copytree(sbom_source, sbom_target)
+
+    archive = output / f"modelforge-{version}.tar.gz"
+    if archive.exists():
+        archive.unlink()
+    # A deterministic archive: sorted entries, and identity metadata normalised so two builds of the
+    # same tree produce the same bytes rather than differing by uid and mtime.
+    def normalise(info: tarfile.TarInfo) -> tarfile.TarInfo:
+        info.uid = info.gid = 0
+        info.uname = info.gname = "root"
+        info.mtime = 0
+        return info
+
+    # gzip writes the current time into its header, so "w:gz" alone produces a different digest on
+    # every build even when the contents are identical. Writing through a GzipFile with mtime=0
+    # makes two builds of the same tree byte-identical, which is what makes a published checksum
+    # worth anything.
+    with (
+        archive.open("wb") as raw,
+        gzip.GzipFile(fileobj=raw, mode="wb", compresslevel=9, mtime=0) as compressed,
+        tarfile.open(fileobj=compressed, mode="w") as tar,
+    ):
+        for path in sorted(staging.rglob("*")):
+            # recursive=False matters: tar.add recurses into a directory by default, so adding the
+            # directory and then each of its children again put every file in the archive several
+            # times over.
+            tar.add(
+                path,
+                arcname=str(path.relative_to(output)),
+                filter=normalise,
+                recursive=False,
+            )
+    shutil.rmtree(staging)
+    return archive
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--output", default="dist")
+    parser.add_argument("--skip-images", action="store_true", help="package without building")
+    parser.add_argument("--allow-dirty", action="store_true")
+    parser.add_argument(
+        "--built-at",
+        default=None,
+        help="fixed ISO-8601 build timestamp; reuse it when proving archive reproducibility",
+    )
+    parser.add_argument(
+        "--public-api-origin",
+        default=os.environ.get("MODELFORGE_PUBLIC_API_ORIGIN", ""),
+        help=(
+            "absolute origin the console will reach its API on, for example "
+            "https://modelforge.example.com. Vite compiles it into the bundle and the CSP is "
+            "derived from it, so a release cannot be corrected here afterwards. Required unless "
+            "--skip-images. Also read from MODELFORGE_PUBLIC_API_ORIGIN."
+        ),
+    )
+    args = parser.parse_args(argv)
+
+    output = (ROOT / args.output).resolve()
+    output.mkdir(parents=True, exist_ok=True)
+    built_at = args.built_at or datetime.now(UTC).isoformat()
+    version = PRODUCT_VERSION
+
+    # Validated before anything is built: refusing after three images is worse than refusing first.
+    public_api_origin = (
+        "" if args.skip_images else normalise_public_api_origin(args.public_api_origin)
+    )
+
+    print(f"{PRODUCT_NAME} {version} — release build", flush=True)
+    source = source_identity(args.allow_dirty)
+    print(f"  source commit {source['commit']}"
+          f"{' (DIRTY)' if source['dirty'] else ''}", flush=True)
+
+    images = (
+        {}
+        if args.skip_images
+        else build_images(version, source["commit"], built_at, public_api_origin)
+    )
+
+    sbom_output = output / "sbom"
+    run(
+        sys.executable,
+        str(ROOT / "scripts" / "m16_sbom.py"),
+        "--output",
+        str(sbom_output),
+        "--version",
+        version,
+        "--source-commit",
+        source["commit"],
+        "--generated-at",
+        built_at,
+    )
+    sbom_artifact = output / f"modelforge-{version}-cyclonedx.json"
+    provenance_artifact = output / f"modelforge-{version}-image-provenance.json"
+    shutil.copy2(sbom_output / "modelforge-cyclonedx.json", sbom_artifact)
+    shutil.copy2(sbom_output / "image-provenance.json", provenance_artifact)
+
+    # Read the artifact, not the source that produced it. v1.2.0 was published with correct
+    # labels, a correct manifest and a console bundle pointing at localhost; nothing upstream of
+    # here could see that, because the defect only exists once the image is built.
+    if images:
+        print("  verifying the console image against its declared API origin", flush=True)
+        acceptance = run(
+            sys.executable,
+            str(ROOT / "scripts" / "release_image_acceptance.py"),
+            "--image",
+            images["modelforge-web"]["tag"],
+            "--expect-origin",
+            public_api_origin,
+            "--version",
+            version,
+            check=False,
+        )
+        for line in acceptance.splitlines():
+            # A release build must not fail because the host console cannot encode a character in
+            # a subprocess's output. Windows defaults to cp1252 here.
+            safe = line.encode(sys.stdout.encoding or "utf-8", "replace").decode(
+                sys.stdout.encoding or "utf-8", "replace"
+            )
+            print(f"  {safe}", flush=True)
+        if "All " not in acceptance:
+            raise SystemExit(
+                "the built console image does not match the API origin it was built for; "
+                "refusing to package a release that cannot reach its own API."
+            )
+
+    archive = package(output, version, sbom_output)
+    print(f"  packaged {archive.name} ({archive.stat().st_size} bytes)", flush=True)
+
+    manifest: dict[str, Any] = {
+        "product": PRODUCT_NAME,
+        "version": version,
+        "channel": RELEASE_CHANNEL,
+        "built_at": built_at,
+        "source": source,
+        "compatibility": {
+            "schema_revision": TARGET_SCHEMA_REVISION,
+            "supported_schema_revisions": list(SUPPORTED_SCHEMA_REVISIONS),
+            "supported_upgrade_source_schema_revisions": list(
+                SUPPORTED_UPGRADE_SOURCE_SCHEMA_REVISIONS
+            ),
+            "agent_protocol_version": CURRENT_AGENT_PROTOCOL_VERSION,
+            "supported_agent_protocol_versions": list(SUPPORTED_AGENT_PROTOCOL_VERSIONS),
+            "minimum_upgrade_source": MINIMUM_UPGRADE_SOURCE,
+            "minimum_postgres_major": MINIMUM_POSTGRES_MAJOR,
+        },
+        "external_dependencies": {
+            "postgresql": f">={MINIMUM_POSTGRES_MAJOR}",
+            "redis": ">=7",
+            "docker_engine": ">=24",
+            "nvidia_container_runtime": "required on any GPU node",
+        },
+        "images": images,
+        # The origin compiled into the console bundle and into its CSP. Recorded because it is a
+        # property of the artifact that an operator otherwise cannot see without unpacking the
+        # image, and because a console pointed at the wrong API is indistinguishable from a
+        # healthy one until someone opens it.
+        "console": {"public_api_origin": public_api_origin or None},
+        "artifacts": {},
+    }
+
+    manifest["sbom"] = {
+        "format": "CycloneDX 1.5",
+        "cyclonedx": sha256_file(sbom_artifact),
+        "provenance": sha256_file(provenance_artifact),
+    }
+
+    for path in sorted(output.glob(f"modelforge-{version}*")):
+        if path.suffix == ".json" or path.name.endswith(".sha256"):
+            continue
+        manifest["artifacts"][path.name] = {
+            "sha256": sha256_file(path),
+            "bytes": path.stat().st_size,
+        }
+
+    manifest_path = output / f"modelforge-{version}-release-manifest.json"
+    # newline="\n" on every release artifact. Python otherwise translates to the host's line ending,
+    # so a Windows-built release and a Linux-built release of the same commit would differ byte for
+    # byte — and the recorded hashes with them.
+    manifest_path.write_text(
+        json.dumps(manifest, indent=2, sort_keys=True), encoding="utf-8", newline="\n"
+    )
+
+    # The checksum file covers the manifest too, so a tampered manifest is as detectable as a
+    # tampered archive.
+    checksums = output / f"modelforge-{version}-SHA256SUMS"
+    lines = [
+        f"{sha256_file(path)}  {path.name}"
+        for path in sorted(output.iterdir())
+        if path.is_file() and not path.name.endswith("SHA256SUMS")
+    ]
+    # Must be LF. `sha256sum -c` treats a trailing CR as part of the filename, so a CRLF checksum
+    # file fails to verify every artifact it covers — on the very command an operator is told to run.
+    checksums.write_text("\n".join(lines) + "\n", encoding="utf-8", newline="\n")
+
+    print(f"  manifest  {manifest_path.name}", flush=True)
+    print(f"  checksums {checksums.name} ({len(lines)} files)", flush=True)
+    for line in lines:
+        print(f"    {line}", flush=True)
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/release_image_acceptance.py

@@ -0,0 +1,215 @@
+"""Prove a built Console release image talks to the API origin it was built for.
+
+This gate exists because v1.2.0 passed every other one and still shipped a console that could not
+reach its own API. Vite inlines ``VITE_API_BASE_URL`` at build time, so the defect lived in the
+*artifact*, not the source: the unit tests were green, the release manifest was correct, the image
+labels were correct, and the bundle inside the image pointed at ``http://localhost:8000``.
+
+Nothing that inspects source, labels or checksums can see that. So this reads the built image.
+
+    python scripts/release_image_acceptance.py --image modelforge-web:1.2.1 \
+        --expect-origin http://192.0.2.10:18000
+
+It is deliberately cheap — it extracts the served assets and the rendered nginx policy from the
+image and asserts on their contents — so it can run in the release gate on any host with Docker and
+without standing up a browser. A full browser pass against a disposable Compose stack remains the
+acceptance step for a deployment; this is the step that stops a wrong artifact being published at
+all.
+
+Exit status is 0 when the image is coherent, 1 when it is not.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import re
+import subprocess  # noqa: S404 - fixed argv, never a shell
+import sys
+from dataclasses import dataclass
+from urllib.parse import urlparse
+
+#: Origins that are never correct in a published release artifact. A console built for a laptop is
+#: fine; a console *published* for a laptop is the v1.2.0 defect.
+DEVELOPMENT_ORIGINS = ("http://localhost:8000", "http://127.0.0.1:8000")
+
+
+@dataclass
+class Check:
+    name: str
+    passed: bool
+    detail: str
+
+
+def run(*args: str) -> str:
+    completed = subprocess.run(
+        list(args), capture_output=True, text=True, encoding="utf-8", errors="replace", check=False
+    )
+    if completed.returncode != 0:
+        raise RuntimeError(f"{' '.join(args)} failed: {completed.stderr.strip()[:400]}")
+    return completed.stdout
+
+
+def image_shell(image: str, script: str) -> str:
+    """Run a read-only shell snippet inside the image without starting the server."""
+
+    return run("docker", "run", "--rm", "--entrypoint", "sh", image, "-c", script)
+
+
+def bundle_origins(image: str) -> set[str]:
+    output = image_shell(
+        image,
+        "grep -ohoE 'https?://[a-zA-Z0-9._:-]+' /usr/share/nginx/html/assets/*.js 2>/dev/null "
+        "| sort -u",
+    )
+    return {line.strip() for line in output.splitlines() if line.strip()}
+
+
+def csp_connect_src(image: str) -> str:
+    output = image_shell(
+        image,
+        "grep -o \"connect-src[^;]*;\" /etc/nginx/conf.d/security-headers.inc 2>/dev/null || true",
+    )
+    return output.strip()
+
+
+def image_labels(image: str) -> dict[str, str]:
+    raw = run("docker", "image", "inspect", "--format", "{{json .Config.Labels}}", image)
+    return json.loads(raw or "{}") or {}
+
+
+def evaluate(image: str, expected: str, version: str | None) -> list[Check]:
+    expected = expected.rstrip("/")
+    parsed = urlparse(expected)
+    checks: list[Check] = []
+
+    origins = bundle_origins(image)
+    checks.append(
+        Check(
+            "bundle targets the configured origin",
+            expected in origins,
+            f"expected {expected}; bundle references {sorted(origins) or 'none'}",
+        )
+    )
+
+    # The whole point of the gate: a development fallback must never reach a published artifact,
+    # unless the artifact is deliberately a development one.
+    leaked = sorted(o for o in DEVELOPMENT_ORIGINS if o in origins and o != expected)
+    checks.append(
+        Check(
+            "bundle contains no development API fallback",
+            not leaked,
+            f"development origins compiled into the bundle: {leaked}" if leaked else "none present",
+        )
+    )
+
+    connect = csp_connect_src(image)
+    checks.append(
+        Check(
+            "CSP connect-src names the same origin",
+            bool(connect) and expected in connect,
+            f"connect-src is {connect!r}",
+        )
+    )
+    directive = connect.split("connect-src", 1)[-1].split(";")[0] if connect else ""
+    checks.append(
+        Check(
+            "CSP connect-src is not widened",
+            "*" not in directive and "data:" not in directive,
+            f"directive is {directive.strip()!r}",
+        )
+    )
+    checks.append(
+        Check(
+            "CSP has no unsubstituted placeholder",
+            "__API_ORIGIN__" not in connect,
+            "placeholder still present" if "__API_ORIGIN__" in connect else "substituted",
+        )
+    )
+
+    # A console that resolves its API relative to itself would make the origin irrelevant; assert
+    # the compiled form actually is absolute, so the CSP coupling above is meaningful.
+    checks.append(
+        Check(
+            "configured origin is absolute",
+            parsed.scheme in {"http", "https"} and bool(parsed.netloc),
+            f"origin parsed as scheme={parsed.scheme!r} netloc={parsed.netloc!r}",
+        )
+    )
+
+    labels = image_labels(image)
+    for field in (
+        "org.opencontainers.image.version",
+        "org.opencontainers.image.revision",
+        "org.opencontainers.image.created",
+        "org.opencontainers.image.source",
+        "org.opencontainers.image.title",
+    ):
+        value = (labels.get(field) or "").strip()
+        checks.append(Check(f"label {field} is populated", bool(value), value or "EMPTY"))
+
+    if version:
+        actual = (labels.get("org.opencontainers.image.version") or "").strip()
+        checks.append(
+            Check(
+                "label version matches the release",
+                actual == version,
+                f"label {actual!r} vs release {version!r}",
+            )
+        )
+    revision = (labels.get("org.opencontainers.image.revision") or "").strip()
+    checks.append(
+        Check(
+            "label revision looks like a commit sha",
+            bool(re.fullmatch(r"[0-9a-f]{40}", revision)),
+            revision or "EMPTY",
+        )
+    )
+    return checks
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--image", required=True, help="the built Console image to inspect")
+    parser.add_argument(
+        "--expect-origin",
+        required=True,
+        help="the API origin this image was built for, for example https://modelforge.example.com",
+    )
+    parser.add_argument("--version", default=None, help="release version the labels must declare")
+    parser.add_argument("--report", default=None, help="write the result as JSON")
+    args = parser.parse_args(argv)
+
+    print(f"Console release-image acceptance: {args.image}", flush=True)
+    checks = evaluate(args.image, args.expect_origin, args.version)
+    for check in checks:
+        print(f"  {'OK  ' if check.passed else 'FAIL'}  {check.name:44} {check.detail}", flush=True)
+
+    failed = [check for check in checks if not check.passed]
+    if args.report:
+        from pathlib import Path
+
+        Path(args.report).write_text(
+            json.dumps(
+                {
+                    "image": args.image,
+                    "expected_origin": args.expect_origin,
+                    "verdict": "PASS" if not failed else "FAIL",
+                    "checks": [vars(check) for check in checks],
+                },
+                indent=2,
+            )
+            + "\n",
+            encoding="utf-8",
+            newline="\n",
+        )
+
+    if failed:
+        print(f"\n{len(failed)} check(s) failed; this image must not be published.", file=sys.stderr)
+        return 1
+    print(f"\nAll {len(checks)} checks passed.", flush=True)
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/upgrade.py

@@ -0,0 +1,310 @@
+"""Upgrade a ModelForge deployment, or refuse and say why.
+
+    python scripts/upgrade.py --plan          # preflight only, changes nothing
+    python scripts/upgrade.py --apply         # preflight, then migrate
+
+The preflight is the point. An upgrade that starts and then discovers it has no recoverable backup,
+or that the schema in front of it is one this release does not understand, has already taken the
+deployment down to learn something it could have known first.
+
+It refuses by default when no verified backup exists. That refusal is overridable, because an
+operator who has taken a backup by other means should not be blocked by a tool that cannot see it —
+but the override is explicit and recorded, never a default.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+import time
+from dataclasses import dataclass, field
+from datetime import UTC, datetime, timedelta
+from pathlib import Path
+from typing import Any
+
+from alembic import command
+from alembic.config import Config
+from sqlalchemy import create_engine, inspect, text
+
+ROOT = Path(__file__).resolve().parents[1]
+sys.path.insert(0, str(ROOT / "backend" / "src"))
+
+from modelforge_api.domain.release import (
+    MINIMUM_POSTGRES_MAJOR,
+    MINIMUM_UPGRADE_SOURCE,
+    PRODUCT_NAME,
+    PRODUCT_VERSION,
+    SUPPORTED_AGENT_PROTOCOL_VERSIONS,
+    SUPPORTED_UPGRADE_SOURCE_SCHEMA_REVISIONS,
+    TARGET_SCHEMA_REVISION,
+    Compatibility,
+    agent_protocol_compatibility,
+    schema_compatibility,
+    upgrade_required,
+)
+
+#: A backup older than this is reported as stale. It does not block on its own — the operator sees
+#: the age and decides — but an upgrade behind a two-day-old recovery point is worth saying out loud.
+BACKUP_FRESHNESS = timedelta(hours=26)
+
+
+@dataclass
+class Finding:
+    name: str
+    ok: bool
+    detail: str
+    blocking: bool = True
+
+
+@dataclass
+class UpgradePlan:
+    findings: list[Finding] = field(default_factory=list)
+
+    def add(self, name: str, ok: bool, detail: str, *, blocking: bool = True) -> None:
+        self.findings.append(Finding(name, ok, detail, blocking))
+
+    @property
+    def blockers(self) -> list[Finding]:
+        return [item for item in self.findings if item.blocking and not item.ok]
+
+    def as_dict(self) -> dict[str, Any]:
+        return {
+            "product": PRODUCT_NAME,
+            "target_version": PRODUCT_VERSION,
+            "target_schema": TARGET_SCHEMA_REVISION,
+            "minimum_upgrade_source": MINIMUM_UPGRADE_SOURCE,
+            "findings": [
+                {
+                    "name": item.name,
+                    "ok": item.ok,
+                    "detail": item.detail,
+                    "blocking": item.blocking,
+                }
+                for item in self.findings
+            ],
+            "blockers": [item.name for item in self.blockers],
+        }
+
+
+def build_plan(database_url: str, *, require_backup: bool) -> UpgradePlan:
+    plan = UpgradePlan()
+    engine = create_engine(database_url, pool_pre_ping=True)
+    try:
+        with engine.connect() as connection:
+            server_version = int(
+                connection.exec_driver_sql("show server_version_num").scalar_one()
+            )
+            has_schema = inspect(connection).has_table("alembic_version")
+            revision = (
+                connection.execute(
+                    text("select version_num from alembic_version")
+                ).scalar_one_or_none()
+                if has_schema
+                else None
+            )
+            major = server_version // 10000
+            plan.add(
+                "postgresql version",
+                major >= MINIMUM_POSTGRES_MAJOR,
+                f"server major {major} (minimum {MINIMUM_POSTGRES_MAJOR})",
+            )
+
+            if revision == TARGET_SCHEMA_REVISION:
+                plan.add(
+                    "current schema",
+                    True,
+                    f"{revision} — already at the target; no migration will run",
+                )
+            elif revision in SUPPORTED_UPGRADE_SOURCE_SCHEMA_REVISIONS:
+                plan.add(
+                    "current schema",
+                    True,
+                    f"supported upgrade source {revision} -> {TARGET_SCHEMA_REVISION}",
+                )
+            else:
+                compatibility = schema_compatibility(revision)
+                plan.add(
+                    "current schema",
+                    False,
+                    f"{revision or '(empty)'} is {compatibility}; "
+                    f"{upgrade_required(compatibility)}",
+                )
+
+            # A database can carry alembic_version without carrying the rest of the schema — a
+            # half-finished migration, or a database that simply is not ModelForge's. Probing it as
+            # though the tables exist turns a preflight into a stack trace, which is the opposite of
+            # what a preflight is for.
+            inspector = inspect(connection)
+            expected_tables = (
+                "backup_sets",
+                "serving_jobs",
+                "compute_nodes",
+                "migration_cutover_operations",
+            )
+            missing_tables = [
+                name for name in expected_tables if not inspector.has_table(name)
+            ]
+            if has_schema and missing_tables:
+                plan.add(
+                    "schema completeness",
+                    False,
+                    f"alembic_version is present but {len(missing_tables)} expected table(s) are "
+                    f"missing ({', '.join(missing_tables)}); this database is either partially "
+                    "migrated or not a ModelForge control plane",
+                )
+
+            if has_schema and not missing_tables:
+                newest = connection.execute(
+                    text(
+                        "select backup_id, created_at from backup_sets "
+                        "where state = 'VERIFIED' order by created_at desc limit 1"
+                    )
+                ).first()
+                if newest is None:
+                    plan.add(
+                        "verified backup",
+                        not require_backup,
+                        "no verified backup exists; an upgrade without a recovery point cannot be "
+                        "undone if the schema turns out to be irreversible",
+                        blocking=require_backup,
+                    )
+                else:
+                    backup_id, created_at = newest
+                    age = datetime.now(UTC) - created_at
+                    plan.add(
+                        "verified backup",
+                        True,
+                        f"{backup_id}, {age.total_seconds() / 3600:.1f} h old",
+                    )
+                    plan.add(
+                        "backup freshness",
+                        age <= BACKUP_FRESHNESS,
+                        f"{age.total_seconds() / 3600:.1f} h old "
+                        f"(advisory threshold {BACKUP_FRESHNESS.total_seconds() / 3600:.0f} h)",
+                        blocking=False,
+                    )
+
+                stale_work = connection.execute(
+                    text(
+                        "select count(*) from serving_jobs "
+                        "where status in ('queued','leased','running')"
+                    )
+                ).scalar_one()
+                plan.add(
+                    "no work in flight",
+                    int(stale_work) == 0,
+                    f"{stale_work} serving job(s) queued, leased or running",
+                    blocking=False,
+                )
+
+                protocols = connection.execute(
+                    text(
+                        "select distinct agent_protocol_version from compute_nodes "
+                        "where enabled and agent_protocol_version is not null"
+                    )
+                ).scalars().all()
+                incompatible = [
+                    version
+                    for version in protocols
+                    if agent_protocol_compatibility(int(version)) is not Compatibility.COMPATIBLE
+                ]
+                plan.add(
+                    "agent protocol",
+                    not incompatible,
+                    f"enabled nodes speak {sorted(protocols) or ['(none reported)']}; "
+                    f"this release supports {list(SUPPORTED_AGENT_PROTOCOL_VERSIONS)}"
+                    + (f"; incompatible: {incompatible}" if incompatible else ""),
+                )
+
+                cutovers = connection.execute(
+                    text(
+                        "select count(*) from migration_cutover_operations "
+                        "where stage not in ('COMMITTED','ROLLED_BACK','ABORTED')"
+                    )
+                ).scalar_one()
+                plan.add(
+                    "no migration cutover mid-flight",
+                    int(cutovers) == 0,
+                    f"{cutovers} cutover(s) in an intermediate stage; these are never "
+                    "auto-resolved and an upgrade will not resolve them either",
+                    blocking=False,
+                )
+    finally:
+        engine.dispose()
+
+    plan.add(
+        "rollback",
+        True,
+        "the schema changes during this upgrade; application rollback requires restoring the "
+        "verified pre-upgrade backup. The 0022 -> 0021 DDL downgrade is rehearsal-only because "
+        "it removes terminal node tombstones and decommission-operation evidence",
+        blocking=False,
+    )
+    return plan
+
+
+def apply_migrations(database_url: str) -> tuple[str | None, str, float]:
+    started = time.perf_counter()
+    engine = create_engine(database_url)
+    with engine.connect() as connection:
+        before = (
+            connection.execute(text("select version_num from alembic_version")).scalar_one_or_none()
+            if inspect(connection).has_table("alembic_version")
+            else None
+        )
+    config = Config(str(ROOT / "backend" / "alembic.ini"))
+    config.set_main_option("script_location", str(ROOT / "backend" / "alembic"))
+    config.set_main_option("sqlalchemy.url", database_url)
+    command.upgrade(config, "head")
+    with engine.connect() as connection:
+        after = connection.execute(text("select version_num from alembic_version")).scalar_one()
+    engine.dispose()
+    return before, str(after), time.perf_counter() - started
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--database-url", required=True)
+    parser.add_argument("--plan", action="store_true", help="preflight only")
+    parser.add_argument("--apply", action="store_true", help="preflight, then migrate")
+    parser.add_argument(
+        "--allow-without-backup",
+        action="store_true",
+        help="proceed with no verified backup; explicit and recorded, never a default",
+    )
+    parser.add_argument("--report", default=None)
+    args = parser.parse_args(argv)
+    if not (args.plan or args.apply):
+        parser.error("choose --plan or --apply")
+
+    print(f"{PRODUCT_NAME} upgrade to {PRODUCT_VERSION}", flush=True)
+    plan = build_plan(args.database_url, require_backup=not args.allow_without_backup)
+    for finding in plan.findings:
+        mark = "OK  " if finding.ok else ("BLOCK" if finding.blocking else "WARN")
+        print(f"  {mark:5} {finding.name:32} {finding.detail}", flush=True)
+
+    report: dict[str, Any] = plan.as_dict()
+    report["applied"] = False
+    if plan.blockers:
+        print(f"\nrefusing to upgrade: {len(plan.blockers)} blocker(s)", flush=True)
+    elif args.apply:
+        before, after, seconds = apply_migrations(args.database_url)
+        report["applied"] = True
+        report["migration"] = {
+            "from": before,
+            "to": after,
+            "seconds": round(seconds, 3),
+            "changed": before != after,
+        }
+        verb = "already current" if before == after else "migrated"
+        print(f"\n  schema {verb}: {before} -> {after} in {seconds:.2f}s", flush=True)
+
+    if args.report:
+        Path(args.report).write_text(json.dumps(report, indent=2), encoding="utf-8")
+        print(f"  report written to {args.report}", flush=True)
+    return 1 if plan.blockers else 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/validate-public-source.mjs

@@ -0,0 +1,94 @@
+#!/usr/bin/env node
+
+import crypto from "node:crypto";
+import fs from "node:fs";
+import path from "node:path";
+import process from "node:process";
+
+function fail(message) {
+  console.error(message);
+  process.exit(1);
+}
+
+function walk(directory, prefix = "") {
+  const files = [];
+  for (const entry of fs.readdirSync(directory, { withFileTypes: true })) {
+    if (!prefix && entry.name === ".git") continue;
+    const relativePath = prefix ? `${prefix}/${entry.name}` : entry.name;
+    const absolutePath = path.join(directory, entry.name);
+    if (entry.isSymbolicLink()) fail(`Symbolic link present in public export: ${relativePath}`);
+    if (entry.isDirectory()) files.push(...walk(absolutePath, relativePath));
+    else if (entry.isFile()) files.push(relativePath);
+    else fail(`Non-regular entry present in public export: ${relativePath}`);
+  }
+  return files;
+}
+
+const required = [
+  "README.md", "SECURITY.md", "CONTRIBUTING.md", "LICENSE", "VERSION",
+  "PUBLIC_SOURCE_EXPORT.md", "PUBLIC_SOURCE_MANIFEST.json", "docker-compose.yml",
+  "backend/pyproject.toml", "frontend/package.json", "node-agent/pyproject.toml",
+  "runtime-worker/pyproject.toml"
+];
+const missing = required.filter((entry) => !fs.existsSync(path.join(process.cwd(), entry)));
+if (missing.length > 0) {
+  fail(`Missing public source files: ${missing.join(", ")}`);
+}
+
+for (const privatePath of [".claude", ".agents", ".codex", "artifacts", "docs/quality"]) {
+  if (fs.existsSync(path.join(process.cwd(), privatePath))) {
+    fail(`Private-only path present in public export: ${privatePath}`);
+  }
+}
+
+const manifest = JSON.parse(fs.readFileSync("PUBLIC_SOURCE_MANIFEST.json", "utf8"));
+if (
+  manifest.schemaVersion !== 1 ||
+  typeof manifest.sourceRevision !== "string" ||
+  !/^[0-9a-f]{40}$/u.test(manifest.sourceRevision) ||
+  !Array.isArray(manifest.files) ||
+  manifest.files.length === 0
+) {
+  fail("PUBLIC_SOURCE_MANIFEST.json has an invalid schema or source revision.");
+}
+
+const declared = new Set();
+for (const entry of manifest.files) {
+  if (
+    !entry ||
+    typeof entry.path !== "string" ||
+    entry.path !== path.posix.normalize(entry.path) ||
+    path.posix.isAbsolute(entry.path) ||
+    entry.path.startsWith("../") ||
+    entry.path.includes("\\") ||
+    !Number.isSafeInteger(entry.bytes) ||
+    entry.bytes < 0 ||
+    typeof entry.sha256 !== "string" ||
+    !/^[0-9a-f]{64}$/u.test(entry.sha256)
+  ) {
+    fail("PUBLIC_SOURCE_MANIFEST.json contains an invalid file entry.");
+  }
+  const collisionKey = entry.path.toLocaleLowerCase("en-US");
+  if (declared.has(collisionKey)) fail(`Duplicate manifest path: ${entry.path}`);
+  declared.add(collisionKey);
+
+  const absolutePath = path.join(process.cwd(), ...entry.path.split("/"));
+  if (!fs.existsSync(absolutePath)) fail(`Manifest file is missing: ${entry.path}`);
+  const stat = fs.lstatSync(absolutePath);
+  if (!stat.isFile() || stat.isSymbolicLink()) {
+    fail(`Manifest path is not a regular file: ${entry.path}`);
+  }
+  const bytes = fs.readFileSync(absolutePath);
+  const digest = crypto.createHash("sha256").update(bytes).digest("hex");
+  if (bytes.length !== entry.bytes) fail(`Manifest size mismatch: ${entry.path}`);
+  if (digest !== entry.sha256) fail(`Manifest hash mismatch: ${entry.path}`);
+}
+
+const generated = new Set(["public_source_export.md", "public_source_manifest.json"]);
+const unexpected = walk(process.cwd()).filter(
+  (entry) => !declared.has(entry.toLocaleLowerCase("en-US")) &&
+    !generated.has(entry.toLocaleLowerCase("en-US"))
+);
+if (unexpected.length > 0) fail(`Unexpected public source files: ${unexpected.join(", ")}`);
+
+console.log(`Public source integrity passed (${manifest.files.length} allowlisted files).`);

scripts/validate_trivy_report.py

@@ -0,0 +1,180 @@
+"""Validate and summarize one Trivy image report without accepting ambiguous JSON.
+
+Trivy returning zero is not sufficient release evidence: an empty, duplicate-key or unrelated
+report can otherwise look like a clean scan. This parser binds the report to the exact image ID,
+requires package coverage, and fails on every fixable or unreviewed HIGH/CRITICAL finding. An
+upstream-unfixed finding is non-blocking only when its complete identity matches an explicit
+reviewed baseline.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+from collections import Counter
+from pathlib import Path
+from typing import Any
+
+
+VulnerabilityIdentity = tuple[str, str, str, str]
+
+
+def reject_duplicate_keys(pairs: list[tuple[str, Any]]) -> dict[str, Any]:
+    result: dict[str, Any] = {}
+    for key, value in pairs:
+        if key in result:
+            raise ValueError(f"duplicate JSON key: {key}")
+        result[key] = value
+    return result
+
+
+def load_reviewed_unfixed(path: Path | None) -> set[VulnerabilityIdentity]:
+    if path is None:
+        return set()
+    data = json.loads(
+        path.read_text(encoding="utf-8"), object_pairs_hook=reject_duplicate_keys
+    )
+    if not isinstance(data, dict) or data.get("schema_version") != 1:
+        raise ValueError("reviewed-unfixed baseline must use schema_version 1")
+    entries = data.get("vulnerabilities")
+    if not isinstance(entries, list):
+        raise ValueError("reviewed-unfixed vulnerabilities must be an array")
+    reviewed: set[VulnerabilityIdentity] = set()
+    for entry in entries:
+        if not isinstance(entry, dict):
+            raise ValueError("reviewed-unfixed entry must be an object")
+        identity = tuple(
+            entry.get(field)
+            for field in ("vulnerability_id", "package", "installed_version", "severity")
+        )
+        if not all(isinstance(value, str) and value for value in identity):
+            raise ValueError("reviewed-unfixed identity fields must be non-empty strings")
+        typed_identity = (identity[0], identity[1], identity[2], identity[3].upper())
+        if typed_identity[3] not in {"HIGH", "CRITICAL"}:
+            raise ValueError("reviewed-unfixed severity must be HIGH or CRITICAL")
+        if typed_identity in reviewed:
+            raise ValueError(f"duplicate reviewed-unfixed identity: {typed_identity!r}")
+        reviewed.add(typed_identity)
+    return reviewed
+
+
+def validate(
+    report: Path,
+    expected_image_id: str,
+    reviewed_unfixed_path: Path | None = None,
+) -> dict[str, Any]:
+    data = json.loads(
+        report.read_text(encoding="utf-8"), object_pairs_hook=reject_duplicate_keys
+    )
+    if not isinstance(data, dict):
+        raise ValueError("Trivy report root must be an object")
+    if data.get("ArtifactType") != "container_image":
+        raise ValueError("Trivy report is not a container-image scan")
+
+    metadata = data.get("Metadata")
+    if not isinstance(metadata, dict) or metadata.get("ImageID") != expected_image_id:
+        observed = metadata.get("ImageID") if isinstance(metadata, dict) else None
+        raise ValueError(
+            f"Trivy report image ID {observed!r} does not match {expected_image_id!r}"
+        )
+
+    results = data.get("Results")
+    if not isinstance(results, list) or not results:
+        raise ValueError("Trivy report has no package result coverage")
+
+    covered_targets: list[str] = []
+    severities: Counter[str] = Counter()
+    reviewed_unfixed = load_reviewed_unfixed(reviewed_unfixed_path)
+    observed_reviewed: set[VulnerabilityIdentity] = set()
+    fixable_high_critical = 0
+    unreviewed_unfixed_high_critical = 0
+    findings = 0
+    for result in results:
+        if not isinstance(result, dict):
+            raise ValueError("Trivy result entry must be an object")
+        target = result.get("Target")
+        result_class = result.get("Class")
+        if isinstance(target, str) and target and result_class in {"os-pkgs", "lang-pkgs"}:
+            covered_targets.append(target)
+        vulnerabilities = result.get("Vulnerabilities")
+        if vulnerabilities is None:
+            continue
+        if not isinstance(vulnerabilities, list):
+            raise ValueError("Trivy Vulnerabilities must be an array or null")
+        for finding in vulnerabilities:
+            if not isinstance(finding, dict):
+                raise ValueError("Trivy vulnerability entry must be an object")
+            severity = finding.get("Severity")
+            if not isinstance(severity, str) or not severity:
+                raise ValueError("Trivy vulnerability is missing Severity")
+            normalized_severity = severity.upper()
+            severities[normalized_severity] += 1
+            findings += 1
+            if normalized_severity not in {"HIGH", "CRITICAL"}:
+                continue
+            vulnerability_id = finding.get("VulnerabilityID")
+            package = finding.get("PkgName")
+            installed_version = finding.get("InstalledVersion")
+            if not all(
+                isinstance(value, str) and value
+                for value in (vulnerability_id, package, installed_version)
+            ):
+                raise ValueError("HIGH/CRITICAL finding is missing its exact identity")
+            fixed_version = finding.get("FixedVersion")
+            if isinstance(fixed_version, str) and fixed_version.strip():
+                fixable_high_critical += 1
+                continue
+            identity = (
+                vulnerability_id,
+                package,
+                installed_version,
+                normalized_severity,
+            )
+            if identity in reviewed_unfixed:
+                observed_reviewed.add(identity)
+            else:
+                unreviewed_unfixed_high_critical += 1
+
+    if not covered_targets:
+        raise ValueError("Trivy report covers no OS or language package target")
+
+    return {
+        "schema_version": 2,
+        "image_id": expected_image_id,
+        "artifact_name": data.get("ArtifactName"),
+        "covered_targets": sorted(set(covered_targets)),
+        "finding_count": findings,
+        "severities": dict(sorted(severities.items())),
+        "high_critical_findings": severities["HIGH"] + severities["CRITICAL"],
+        "reviewed_unfixed_high_critical": len(observed_reviewed),
+        "fixable_high_critical": fixable_high_critical,
+        "unreviewed_unfixed_high_critical": unreviewed_unfixed_high_critical,
+        "stale_reviewed_unfixed_entries": len(reviewed_unfixed - observed_reviewed),
+        "release_blockers": fixable_high_critical + unreviewed_unfixed_high_critical,
+    }
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--report", type=Path, required=True)
+    parser.add_argument("--image-id", required=True)
+    parser.add_argument("--summary", type=Path, required=True)
+    parser.add_argument("--reviewed-unfixed", type=Path)
+    args = parser.parse_args()
+
+    summary = validate(args.report, args.image_id, args.reviewed_unfixed)
+    args.summary.parent.mkdir(parents=True, exist_ok=True)
+    args.summary.write_text(
+        json.dumps(summary, indent=2, sort_keys=True) + "\n", encoding="utf-8", newline="\n"
+    )
+    print(
+        f"Trivy coverage: {len(summary['covered_targets'])} target(s), "
+        f"{summary['finding_count']} finding(s), "
+        f"{summary['reviewed_unfixed_high_critical']} reviewed upstream-unfixed, "
+        f"{summary['release_blockers']} HIGH/CRITICAL blocker(s)"
+    )
+    return 1 if summary["release_blockers"] else 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())