ModelForge/scripts/release_build.py

"""Build a ModelForge release: validate, build, inventory, sign the inventory, package.

One command, so that what ships is what was tested rather than whatever happened to be in the
working tree. Everything it emits is derived from the repository and the built images — nothing is
typed in by hand, because a release manifest an operator cannot verify is decoration.

    python scripts/release_build.py --output dist

The build refuses to run against a dirty working tree unless told otherwise. A release built from
uncommitted changes cannot be reproduced, and its recorded source commit would be a lie.
"""

from __future__ import annotations

import argparse
import gzip
import hashlib
import json
import os
import shutil
import subprocess
import sys
import tarfile
import tempfile
from datetime import UTC, datetime
from pathlib import Path
from typing import Any
from urllib.parse import urlparse

ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "backend" / "src"))

from modelforge_api.domain.release import (
    CURRENT_AGENT_PROTOCOL_VERSION,
    MINIMUM_POSTGRES_MAJOR,
    MINIMUM_UPGRADE_SOURCE,
    PRODUCT_NAME,
    PRODUCT_VERSION,
    RELEASE_CHANNEL,
    SUPPORTED_AGENT_PROTOCOL_VERSIONS,
    SUPPORTED_SCHEMA_REVISIONS,
    SUPPORTED_UPGRADE_SOURCE_SCHEMA_REVISIONS,
    TARGET_SCHEMA_REVISION,
)

#: image name -> (build context, Dockerfile path), both relative to the repository root. The Node
#: Agent and Runtime Worker build from the repository root because their Dockerfiles copy a
#: component directory from that shared context. Keeping the four distributable images in one
#: inventory makes it impossible for the private runtime boundary to disappear from a release.
IMAGES = {
    "modelforge-api": ("backend", "backend/Dockerfile"),
    "modelforge-web": ("frontend", "frontend/Dockerfile"),
    "modelforge-node-agent": (".", "node-agent/Dockerfile"),
    "modelforge-runtime-worker": (".", "runtime-worker/Dockerfile"),
}

#: What a release tarball contains. Deliberately no model weights, no .env, no database dump and no
#: credential of any kind — see the packaging test, which fails if that changes.
ARTIFACT_PATHS = (
    "docker-compose.yml",
    "docker-compose.production.yml",
    "docker-compose.node-agent.yml",
    "docker-compose.runtime-worker.yml",
    "docker-compose.gpu.yml",
    "docker-compose.backup.yml",
    "docker-compose.dr.yml",
    ".env.example",
    "VERSION",
    "README.md",
    "config",
    "docs",
    "scripts/bootstrap.py",
    "scripts/preflight.py",
)


def run(*args: str, check: bool = True, cwd: Path | None = None) -> str:
    completed = subprocess.run(
        list(args),
        capture_output=True,
        text=True,
        encoding="utf-8",
        errors="replace",
        cwd=cwd or ROOT,
        check=False,
    )
    if check and completed.returncode != 0:
        raise RuntimeError(f"{' '.join(args)} failed: {completed.stderr.strip()[:500]}")
    return (completed.stdout or "").strip()


def sha256_file(path: Path) -> str:
    digest = hashlib.sha256()
    with path.open("rb") as handle:
        for chunk in iter(lambda: handle.read(1024 * 1024), b""):
            digest.update(chunk)
    return digest.hexdigest()


def source_identity(allow_dirty: bool) -> dict[str, Any]:
    commit = run("git", "rev-parse", "HEAD")
    dirty = bool(run("git", "status", "--porcelain"))
    if dirty and not allow_dirty:
        raise SystemExit(
            "refusing to build a release from a dirty working tree: the recorded source commit "
            "would not describe what was built. Commit first, or pass --allow-dirty for a "
            "rehearsal build."
        )
    describe = run("git", "describe", "--tags", "--always", check=False)
    return {
        "commit": commit,
        "describe": describe or None,
        "dirty": dirty,
        "repository": run("git", "remote", "get-url", "origin", check=False) or None,
        "branch": run("git", "rev-parse", "--abbrev-ref", "HEAD", check=False) or None,
    }


def source_date_epoch(built_at: str) -> int:
    """Translate the declared build time into BuildKit's reproducible timestamp contract."""

    parsed = datetime.fromisoformat(built_at.replace("Z", "+00:00"))
    if parsed.tzinfo is None:
        raise ValueError("build timestamp must include a UTC offset")
    return int(parsed.timestamp())


def manifest_digest(repo_digest: str | None) -> str | None:
    """Return the pullable OCI digest rather than Docker's local config/image identifier."""

    if not repo_digest or "@" not in repo_digest:
        return None
    digest = repo_digest.rsplit("@", 1)[1]
    return digest if digest.startswith("sha256:") else None


#: Images whose behaviour depends on the origin the console is served against. Vite inlines
#: VITE_* variables into the bundle at build time, so this is a property of the artifact rather
#: than of the deployment that runs it.
ORIGIN_DEPENDENT_IMAGES = frozenset({"modelforge-web"})


def normalise_public_api_origin(value: str) -> str:
    """Validate the console's compiled-in API origin, or refuse to build a release without one.

    v1.2.0 shipped a console that could not reach its own API. The release build never passed
    ``VITE_API_BASE_URL``, so Vite compiled the Dockerfile's development default into an immutable
    bundle, and the nginx CSP — derived from the same argument — hardcoded the same wrong origin.
    A convenience default is exactly right for ``docker compose up`` on a laptop and exactly wrong
    for a release artifact, so the release path refuses to guess.
    """

    origin = value.strip().rstrip("/")
    if not origin:
        raise SystemExit(
            "a release build needs the public API origin the console will be served against. "
            "Pass --public-api-origin, or set MODELFORGE_PUBLIC_API_ORIGIN. It is compiled into "
            "the bundle and into the CSP, so it cannot be corrected after the fact by the "
            "deployment that runs the image."
        )
    parsed = urlparse(origin)
    if parsed.scheme not in {"http", "https"} or not parsed.netloc:
        raise SystemExit(
            f"public API origin must be an absolute http(s) origin, got {value!r}. "
            "Example: https://modelforge.example.com or http://192.0.2.10:18000"
        )
    if parsed.path or parsed.query or parsed.fragment:
        raise SystemExit(
            f"public API origin must be a bare origin with no path, query or fragment, "
            f"got {value!r}. The console appends its own /api/v1 paths."
        )
    return origin


def build_images(
    version: str, commit: str, built_at: str, public_api_origin: str
) -> dict[str, dict[str, Any]]:
    built: dict[str, dict[str, Any]] = {}
    epoch = source_date_epoch(built_at)
    for image, (context, dockerfile) in IMAGES.items():
        tag = f"{image}:{version}"
        print(f"  building {tag}", flush=True)
        origin_args: tuple[str, ...] = ()
        if image in ORIGIN_DEPENDENT_IMAGES:
            origin_args = ("--build-arg", f"VITE_API_BASE_URL={public_api_origin}")
            print(f"    public API origin {public_api_origin}", flush=True)
        with tempfile.TemporaryDirectory(prefix="modelforge-build-") as temporary:
            metadata_path = Path(temporary) / "metadata.json"
            run(
                "docker",
                "build",
                # BuildKit attestations carry an exporter invocation timestamp and make the local
                # manifest-list ID vary. ModelForge emits its own commit-bound CycloneDX and image
                # provenance below, so suppress the duplicate nondeterministic attestations.
                "--provenance=false",
                "--sbom=false",
                "--metadata-file",
                str(metadata_path),
                "-t",
                tag,
                "-f",
                str(ROOT / dockerfile),
                "--build-arg",
                f"MODELFORGE_VERSION={version}",
                "--build-arg",
                f"MODELFORGE_COMMIT={commit}",
                "--build-arg",
                f"MODELFORGE_BUILT_AT={built_at}",
                "--build-arg",
                f"SOURCE_DATE_EPOCH={epoch}",
                *origin_args,
                str(ROOT / context),
            )
            build_metadata = json.loads(metadata_path.read_text(encoding="utf-8"))
        image_id = run("docker", "inspect", "--format", "{{.Id}}", tag)
        architecture = run("docker", "inspect", "--format", "{{.Architecture}}", tag)
        labels = json.loads(
            run("docker", "inspect", "--format", "{{json .Config.Labels}}", tag) or "{}"
        )
        repo_digest = (
            run("docker", "inspect", "--format", "{{index .RepoDigests 0}}", tag, check=False)
            or None
        )
        built[image] = {
            "tag": tag,
            "image_id": image_id,
            "image_digest": manifest_digest(repo_digest)
            or build_metadata.get("containerimage.digest"),
            "repo_digest": repo_digest,
            "architecture": architecture,
            "labels": {
                key: value
                for key, value in (labels or {}).items()
                if key.startswith("org.opencontainers.image.")
            },
        }
    return built


def package(output: Path, version: str, sbom_source: Path | None = None) -> Path:
    """Assemble the release tarball from tracked paths only."""

    staging = output / f"modelforge-{version}"
    if staging.exists():
        shutil.rmtree(staging)
    staging.mkdir(parents=True)
    for relative in ARTIFACT_PATHS:
        source = ROOT / relative
        if not source.exists():
            print(f"  note: {relative} is absent and was not packaged", flush=True)
            continue
        target = staging / relative
        target.parent.mkdir(parents=True, exist_ok=True)
        if source.is_dir():
            shutil.copytree(source, target, ignore=shutil.ignore_patterns("__pycache__", "*.pyc"))
        else:
            shutil.copy2(source, target)

    if sbom_source is not None:
        sbom_target = staging / "docs" / "security" / "sbom"
        if sbom_target.exists():
            shutil.rmtree(sbom_target)
        shutil.copytree(sbom_source, sbom_target)

    archive = output / f"modelforge-{version}.tar.gz"
    if archive.exists():
        archive.unlink()
    # A deterministic archive: sorted entries, and identity metadata normalised so two builds of the
    # same tree produce the same bytes rather than differing by uid and mtime.
    def normalise(info: tarfile.TarInfo) -> tarfile.TarInfo:
        info.uid = info.gid = 0
        info.uname = info.gname = "root"
        info.mtime = 0
        return info

    # gzip writes the current time into its header, so "w:gz" alone produces a different digest on
    # every build even when the contents are identical. Writing through a GzipFile with mtime=0
    # makes two builds of the same tree byte-identical, which is what makes a published checksum
    # worth anything.
    with (
        archive.open("wb") as raw,
        gzip.GzipFile(fileobj=raw, mode="wb", compresslevel=9, mtime=0) as compressed,
        tarfile.open(fileobj=compressed, mode="w") as tar,
    ):
        for path in sorted(staging.rglob("*")):
            # recursive=False matters: tar.add recurses into a directory by default, so adding the
            # directory and then each of its children again put every file in the archive several
            # times over.
            tar.add(
                path,
                arcname=str(path.relative_to(output)),
                filter=normalise,
                recursive=False,
            )
    shutil.rmtree(staging)
    return archive


def main(argv: list[str] | None = None) -> int:
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("--output", default="dist")
    parser.add_argument("--skip-images", action="store_true", help="package without building")
    parser.add_argument("--allow-dirty", action="store_true")
    parser.add_argument(
        "--built-at",
        default=None,
        help="fixed ISO-8601 build timestamp; reuse it when proving archive reproducibility",
    )
    parser.add_argument(
        "--public-api-origin",
        default=os.environ.get("MODELFORGE_PUBLIC_API_ORIGIN", ""),
        help=(
            "absolute origin the console will reach its API on, for example "
            "https://modelforge.example.com. Vite compiles it into the bundle and the CSP is "
            "derived from it, so a release cannot be corrected here afterwards. Required unless "
            "--skip-images. Also read from MODELFORGE_PUBLIC_API_ORIGIN."
        ),
    )
    args = parser.parse_args(argv)

    output = (ROOT / args.output).resolve()
    output.mkdir(parents=True, exist_ok=True)
    built_at = args.built_at or datetime.now(UTC).isoformat()
    version = PRODUCT_VERSION

    # Validated before anything is built: refusing after three images is worse than refusing first.
    public_api_origin = (
        "" if args.skip_images else normalise_public_api_origin(args.public_api_origin)
    )

    print(f"{PRODUCT_NAME} {version} — release build", flush=True)
    source = source_identity(args.allow_dirty)
    print(f"  source commit {source['commit']}"
          f"{' (DIRTY)' if source['dirty'] else ''}", flush=True)

    images = (
        {}
        if args.skip_images
        else build_images(version, source["commit"], built_at, public_api_origin)
    )

    sbom_output = output / "sbom"
    run(
        sys.executable,
        str(ROOT / "scripts" / "m16_sbom.py"),
        "--output",
        str(sbom_output),
        "--version",
        version,
        "--source-commit",
        source["commit"],
        "--generated-at",
        built_at,
    )
    sbom_artifact = output / f"modelforge-{version}-cyclonedx.json"
    provenance_artifact = output / f"modelforge-{version}-image-provenance.json"
    shutil.copy2(sbom_output / "modelforge-cyclonedx.json", sbom_artifact)
    shutil.copy2(sbom_output / "image-provenance.json", provenance_artifact)

    # Read the artifact, not the source that produced it. v1.2.0 was published with correct
    # labels, a correct manifest and a console bundle pointing at localhost; nothing upstream of
    # here could see that, because the defect only exists once the image is built.
    if images:
        print("  verifying the console image against its declared API origin", flush=True)
        acceptance = run(
            sys.executable,
            str(ROOT / "scripts" / "release_image_acceptance.py"),
            "--image",
            images["modelforge-web"]["tag"],
            "--expect-origin",
            public_api_origin,
            "--version",
            version,
            check=False,
        )
        for line in acceptance.splitlines():
            # A release build must not fail because the host console cannot encode a character in
            # a subprocess's output. Windows defaults to cp1252 here.
            safe = line.encode(sys.stdout.encoding or "utf-8", "replace").decode(
                sys.stdout.encoding or "utf-8", "replace"
            )
            print(f"  {safe}", flush=True)
        if "All " not in acceptance:
            raise SystemExit(
                "the built console image does not match the API origin it was built for; "
                "refusing to package a release that cannot reach its own API."
            )

    archive = package(output, version, sbom_output)
    print(f"  packaged {archive.name} ({archive.stat().st_size} bytes)", flush=True)

    manifest: dict[str, Any] = {
        "product": PRODUCT_NAME,
        "version": version,
        "channel": RELEASE_CHANNEL,
        "built_at": built_at,
        "source": source,
        "compatibility": {
            "schema_revision": TARGET_SCHEMA_REVISION,
            "supported_schema_revisions": list(SUPPORTED_SCHEMA_REVISIONS),
            "supported_upgrade_source_schema_revisions": list(
                SUPPORTED_UPGRADE_SOURCE_SCHEMA_REVISIONS
            ),
            "agent_protocol_version": CURRENT_AGENT_PROTOCOL_VERSION,
            "supported_agent_protocol_versions": list(SUPPORTED_AGENT_PROTOCOL_VERSIONS),
            "minimum_upgrade_source": MINIMUM_UPGRADE_SOURCE,
            "minimum_postgres_major": MINIMUM_POSTGRES_MAJOR,
        },
        "external_dependencies": {
            "postgresql": f">={MINIMUM_POSTGRES_MAJOR}",
            "redis": ">=7",
            "docker_engine": ">=24",
            "nvidia_container_runtime": "required on any GPU node",
        },
        "images": images,
        # The origin compiled into the console bundle and into its CSP. Recorded because it is a
        # property of the artifact that an operator otherwise cannot see without unpacking the
        # image, and because a console pointed at the wrong API is indistinguishable from a
        # healthy one until someone opens it.
        "console": {"public_api_origin": public_api_origin or None},
        "artifacts": {},
    }

    manifest["sbom"] = {
        "format": "CycloneDX 1.5",
        "cyclonedx": sha256_file(sbom_artifact),
        "provenance": sha256_file(provenance_artifact),
    }

    for path in sorted(output.glob(f"modelforge-{version}*")):
        if path.suffix == ".json" or path.name.endswith(".sha256"):
            continue
        manifest["artifacts"][path.name] = {
            "sha256": sha256_file(path),
            "bytes": path.stat().st_size,
        }

    manifest_path = output / f"modelforge-{version}-release-manifest.json"
    # newline="\n" on every release artifact. Python otherwise translates to the host's line ending,
    # so a Windows-built release and a Linux-built release of the same commit would differ byte for
    # byte — and the recorded hashes with them.
    manifest_path.write_text(
        json.dumps(manifest, indent=2, sort_keys=True), encoding="utf-8", newline="\n"
    )

    # The checksum file covers the manifest too, so a tampered manifest is as detectable as a
    # tampered archive.
    checksums = output / f"modelforge-{version}-SHA256SUMS"
    lines = [
        f"{sha256_file(path)}  {path.name}"
        for path in sorted(output.iterdir())
        if path.is_file() and not path.name.endswith("SHA256SUMS")
    ]
    # Must be LF. `sha256sum -c` treats a trailing CR as part of the filename, so a CRLF checksum
    # file fails to verify every artifact it covers — on the very command an operator is told to run.
    checksums.write_text("\n".join(lines) + "\n", encoding="utf-8", newline="\n")

    print(f"  manifest  {manifest_path.name}", flush=True)
    print(f"  checksums {checksums.name} ({len(lines)} files)", flush=True)
    for line in lines:
        print(f"    {line}", flush=True)
    return 0


if __name__ == "__main__":
    raise SystemExit(main())