"""Check a host before installing ModelForge, and say exactly what is wrong. Run this first on a clean machine. It answers the question an operator actually has — "will this work here?" — before anything is downloaded, built or started, and it never changes the system it is inspecting. python scripts/preflight.py python scripts/preflight.py --production # also check the production configuration rules Exit status is 0 when the host can run ModelForge, 1 when something must be fixed. """ from __future__ import annotations import argparse import json import os import shutil import socket import subprocess # noqa: S404 - fixed argv, never a shell import sys from dataclasses import dataclass from pathlib import Path ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT / "backend" / "src")) from modelforge_api.domain.release import ( # noqa: E402 MINIMUM_POSTGRES_MAJOR, PRODUCT_NAME, PRODUCT_VERSION, ) MINIMUM_DOCKER_MAJOR = 24 MINIMUM_COMPOSE_MAJOR = 2 RECOMMENDED_FREE_BYTES = 50 * 1024**3 #: Ports the deployment publishes, and the variable that moves each one. PUBLISHED_PORTS = { "MODELFORGE_API_PUBLISHED_PORT": 8000, "MODELFORGE_WEB_PORT": 3000, "MODELFORGE_POSTGRES_PORT": 5432, "MODELFORGE_REDIS_PORT": 6379, } @dataclass class Check: name: str ok: bool detail: str fatal: bool = True def run(*args: str) -> tuple[int, str]: try: completed = subprocess.run( # noqa: S603 - fixed argv, no shell list(args), capture_output=True, text=True, encoding="utf-8", errors="replace", timeout=60, check=False, ) except (OSError, subprocess.TimeoutExpired) as error: return 1, f"{type(error).__name__}" return completed.returncode, (completed.stdout or completed.stderr or "").strip() def _major(value: str) -> int: digits = "" for character in value.lstrip("v"): if character.isdigit(): digits += character else: break return int(digits) if digits else 0 def check_docker() -> list[Check]: checks: list[Check] = [] if shutil.which("docker") is None: return [Check("docker", False, "docker is not on PATH")] code, version = run("docker", "version", "--format", "{{.Server.Version}}") if code != 0: return [Check("docker", False, f"the Docker daemon is not reachable: {version[:120]}")] major = _major(version) checks.append( Check( "docker engine", major >= MINIMUM_DOCKER_MAJOR, f"{version} (minimum {MINIMUM_DOCKER_MAJOR})", ) ) code, compose = run("docker", "compose", "version", "--short") checks.append( Check( "docker compose", code == 0 and _major(compose) >= MINIMUM_COMPOSE_MAJOR, f"{compose or 'absent'} (minimum {MINIMUM_COMPOSE_MAJOR}.x)", ) ) return checks def check_gpu() -> list[Check]: """A GPU is required on a compute node, not on the control-plane host.""" if shutil.which("nvidia-smi") is None: return [ Check( "nvidia driver", True, "nvidia-smi is absent; required only on a GPU compute node", fatal=False, ) ] code, output = run("nvidia-smi", "--query-gpu=name,driver_version", "--format=csv,noheader") if code != 0: return [Check("nvidia driver", False, "nvidia-smi failed", fatal=False)] return [Check("nvidia driver", True, output.replace("\n", "; "), fatal=False)] def check_ports() -> list[Check]: checks = [] for variable, default in PUBLISHED_PORTS.items(): port = int(os.environ.get(variable, default)) connection = socket.socket() connection.settimeout(1) try: connection.connect(("127.0.0.1", port)) occupied = True except OSError: occupied = False finally: connection.close() checks.append( Check( f"port {port}", not occupied, f"already in use; move it with {variable}" if occupied else "free", fatal=False, ) ) return checks def check_storage(path: Path) -> list[Check]: try: usage = shutil.disk_usage(path) except OSError as error: return [Check("storage", False, f"{path} is not readable: {error}")] free_gib = usage.free / 1024**3 return [ Check( "free storage", usage.free >= RECOMMENDED_FREE_BYTES, f"{free_gib:.1f} GiB free at {path} " f"(recommended {RECOMMENDED_FREE_BYTES / 1024**3:.0f} GiB for model artifacts)", fatal=False, ) ] def check_configuration(production: bool) -> list[Check]: env_file = ROOT / ".env" example = ROOT / ".env.example" checks = [ Check(".env.example", example.is_file(), "present" if example.is_file() else "missing") ] if not env_file.is_file(): checks.append( Check( ".env", not production, "absent; copy .env.example to .env and fill in the generated secrets", fatal=production, ) ) return checks checks.append(Check(".env", True, "present")) if production: from modelforge_api.services.startup_validation import ( StartupFailureCode, validate_settings, ) from modelforge_api.settings import Settings settings = Settings(_env_file=str(env_file)) # type: ignore[call-arg] report = validate_settings(settings) # Storage roots are container paths. Validating them against the host is meaningless — the # host has no /data/artifacts and is not supposed to — so they are checked inside the # container at startup, which is the only place the answer means anything. relevant = [ problem for problem in report.problems if problem.code is not StartupFailureCode.INVALID_STORAGE_ROOT ] for problem in relevant: checks.append(Check(problem.setting, False, f"{problem.code}: {problem.message}")) if not relevant: checks.append( Check( "production configuration", True, "no problems found; storage roots are validated inside the container", ) ) return checks def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--production", action="store_true") parser.add_argument("--storage-path", default=str(ROOT)) parser.add_argument("--json", action="store_true") args = parser.parse_args(argv) checks: list[Check] = [] checks.append( Check( "python", sys.version_info >= (3, 12), f"{sys.version.split()[0]} (minimum 3.12)", fatal=False, ) ) checks += check_docker() checks += check_gpu() checks += check_ports() checks += check_storage(Path(args.storage_path)) checks += check_configuration(args.production) checks.append( Check( "postgresql", True, f"provided by the compose deployment; minimum major {MINIMUM_POSTGRES_MAJOR}", fatal=False, ) ) blocking = [check for check in checks if not check.ok and check.fatal] advisory = [check for check in checks if not check.ok and not check.fatal] if args.json: print( json.dumps( { "product": PRODUCT_NAME, "version": PRODUCT_VERSION, "ready": not blocking, "checks": [ { "name": check.name, "ok": check.ok, "detail": check.detail, "fatal": check.fatal, } for check in checks ], }, indent=2, ) ) else: print(f"{PRODUCT_NAME} {PRODUCT_VERSION} — host preflight") for check in checks: mark = "OK " if check.ok else ("FAIL" if check.fatal else "WARN") print(f" {mark} {check.name:24} {check.detail}") print() if blocking: print(f"{len(blocking)} problem(s) must be fixed before installing.") elif advisory: print(f"Host is ready. {len(advisory)} advisory note(s) above.") else: print("Host is ready.") return 1 if blocking else 0 if __name__ == "__main__": raise SystemExit(main())