283 lines
8.8 KiB
Python
283 lines
8.8 KiB
Python
"""Check a host before installing ModelForge, and say exactly what is wrong.
|
|
|
|
Run this first on a clean machine. It answers the question an operator actually has — "will this
|
|
work here?" — before anything is downloaded, built or started, and it never changes the system it
|
|
is inspecting.
|
|
|
|
python scripts/preflight.py
|
|
python scripts/preflight.py --production # also check the production configuration rules
|
|
|
|
Exit status is 0 when the host can run ModelForge, 1 when something must be fixed.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
import os
|
|
import shutil
|
|
import socket
|
|
import subprocess # noqa: S404 - fixed argv, never a shell
|
|
import sys
|
|
from dataclasses import dataclass
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT / "backend" / "src"))
|
|
|
|
from modelforge_api.domain.release import ( # noqa: E402
|
|
MINIMUM_POSTGRES_MAJOR,
|
|
PRODUCT_NAME,
|
|
PRODUCT_VERSION,
|
|
)
|
|
|
|
MINIMUM_DOCKER_MAJOR = 24
|
|
MINIMUM_COMPOSE_MAJOR = 2
|
|
RECOMMENDED_FREE_BYTES = 50 * 1024**3
|
|
|
|
#: Ports the deployment publishes, and the variable that moves each one.
|
|
PUBLISHED_PORTS = {
|
|
"MODELFORGE_API_PUBLISHED_PORT": 8000,
|
|
"MODELFORGE_WEB_PORT": 3000,
|
|
"MODELFORGE_POSTGRES_PORT": 5432,
|
|
"MODELFORGE_REDIS_PORT": 6379,
|
|
}
|
|
|
|
|
|
@dataclass
|
|
class Check:
|
|
name: str
|
|
ok: bool
|
|
detail: str
|
|
fatal: bool = True
|
|
|
|
|
|
def run(*args: str) -> tuple[int, str]:
|
|
try:
|
|
completed = subprocess.run( # noqa: S603 - fixed argv, no shell
|
|
list(args),
|
|
capture_output=True,
|
|
text=True,
|
|
encoding="utf-8",
|
|
errors="replace",
|
|
timeout=60,
|
|
check=False,
|
|
)
|
|
except (OSError, subprocess.TimeoutExpired) as error:
|
|
return 1, f"{type(error).__name__}"
|
|
return completed.returncode, (completed.stdout or completed.stderr or "").strip()
|
|
|
|
|
|
def _major(value: str) -> int:
|
|
digits = ""
|
|
for character in value.lstrip("v"):
|
|
if character.isdigit():
|
|
digits += character
|
|
else:
|
|
break
|
|
return int(digits) if digits else 0
|
|
|
|
|
|
def check_docker() -> list[Check]:
|
|
checks: list[Check] = []
|
|
if shutil.which("docker") is None:
|
|
return [Check("docker", False, "docker is not on PATH")]
|
|
code, version = run("docker", "version", "--format", "{{.Server.Version}}")
|
|
if code != 0:
|
|
return [Check("docker", False, f"the Docker daemon is not reachable: {version[:120]}")]
|
|
major = _major(version)
|
|
checks.append(
|
|
Check(
|
|
"docker engine",
|
|
major >= MINIMUM_DOCKER_MAJOR,
|
|
f"{version} (minimum {MINIMUM_DOCKER_MAJOR})",
|
|
)
|
|
)
|
|
code, compose = run("docker", "compose", "version", "--short")
|
|
checks.append(
|
|
Check(
|
|
"docker compose",
|
|
code == 0 and _major(compose) >= MINIMUM_COMPOSE_MAJOR,
|
|
f"{compose or 'absent'} (minimum {MINIMUM_COMPOSE_MAJOR}.x)",
|
|
)
|
|
)
|
|
return checks
|
|
|
|
|
|
def check_gpu() -> list[Check]:
|
|
"""A GPU is required on a compute node, not on the control-plane host."""
|
|
|
|
if shutil.which("nvidia-smi") is None:
|
|
return [
|
|
Check(
|
|
"nvidia driver",
|
|
True,
|
|
"nvidia-smi is absent; required only on a GPU compute node",
|
|
fatal=False,
|
|
)
|
|
]
|
|
code, output = run("nvidia-smi", "--query-gpu=name,driver_version", "--format=csv,noheader")
|
|
if code != 0:
|
|
return [Check("nvidia driver", False, "nvidia-smi failed", fatal=False)]
|
|
return [Check("nvidia driver", True, output.replace("\n", "; "), fatal=False)]
|
|
|
|
|
|
def check_ports() -> list[Check]:
|
|
checks = []
|
|
for variable, default in PUBLISHED_PORTS.items():
|
|
port = int(os.environ.get(variable, default))
|
|
connection = socket.socket()
|
|
connection.settimeout(1)
|
|
try:
|
|
connection.connect(("127.0.0.1", port))
|
|
occupied = True
|
|
except OSError:
|
|
occupied = False
|
|
finally:
|
|
connection.close()
|
|
checks.append(
|
|
Check(
|
|
f"port {port}",
|
|
not occupied,
|
|
f"already in use; move it with {variable}" if occupied else "free",
|
|
fatal=False,
|
|
)
|
|
)
|
|
return checks
|
|
|
|
|
|
def check_storage(path: Path) -> list[Check]:
|
|
try:
|
|
usage = shutil.disk_usage(path)
|
|
except OSError as error:
|
|
return [Check("storage", False, f"{path} is not readable: {error}")]
|
|
free_gib = usage.free / 1024**3
|
|
return [
|
|
Check(
|
|
"free storage",
|
|
usage.free >= RECOMMENDED_FREE_BYTES,
|
|
f"{free_gib:.1f} GiB free at {path} "
|
|
f"(recommended {RECOMMENDED_FREE_BYTES / 1024**3:.0f} GiB for model artifacts)",
|
|
fatal=False,
|
|
)
|
|
]
|
|
|
|
|
|
def check_configuration(production: bool) -> list[Check]:
|
|
env_file = ROOT / ".env"
|
|
example = ROOT / ".env.example"
|
|
checks = [
|
|
Check(".env.example", example.is_file(), "present" if example.is_file() else "missing")
|
|
]
|
|
if not env_file.is_file():
|
|
checks.append(
|
|
Check(
|
|
".env",
|
|
not production,
|
|
"absent; copy .env.example to .env and fill in the generated secrets",
|
|
fatal=production,
|
|
)
|
|
)
|
|
return checks
|
|
checks.append(Check(".env", True, "present"))
|
|
if production:
|
|
from modelforge_api.services.startup_validation import (
|
|
StartupFailureCode,
|
|
validate_settings,
|
|
)
|
|
from modelforge_api.settings import Settings
|
|
|
|
settings = Settings(_env_file=str(env_file)) # type: ignore[call-arg]
|
|
report = validate_settings(settings)
|
|
# Storage roots are container paths. Validating them against the host is meaningless — the
|
|
# host has no /data/artifacts and is not supposed to — so they are checked inside the
|
|
# container at startup, which is the only place the answer means anything.
|
|
relevant = [
|
|
problem
|
|
for problem in report.problems
|
|
if problem.code is not StartupFailureCode.INVALID_STORAGE_ROOT
|
|
]
|
|
for problem in relevant:
|
|
checks.append(Check(problem.setting, False, f"{problem.code}: {problem.message}"))
|
|
if not relevant:
|
|
checks.append(
|
|
Check(
|
|
"production configuration",
|
|
True,
|
|
"no problems found; storage roots are validated inside the container",
|
|
)
|
|
)
|
|
return checks
|
|
|
|
|
|
def main(argv: list[str] | None = None) -> int:
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--production", action="store_true")
|
|
parser.add_argument("--storage-path", default=str(ROOT))
|
|
parser.add_argument("--json", action="store_true")
|
|
args = parser.parse_args(argv)
|
|
|
|
checks: list[Check] = []
|
|
checks.append(
|
|
Check(
|
|
"python",
|
|
sys.version_info >= (3, 12),
|
|
f"{sys.version.split()[0]} (minimum 3.12)",
|
|
fatal=False,
|
|
)
|
|
)
|
|
checks += check_docker()
|
|
checks += check_gpu()
|
|
checks += check_ports()
|
|
checks += check_storage(Path(args.storage_path))
|
|
checks += check_configuration(args.production)
|
|
checks.append(
|
|
Check(
|
|
"postgresql",
|
|
True,
|
|
f"provided by the compose deployment; minimum major {MINIMUM_POSTGRES_MAJOR}",
|
|
fatal=False,
|
|
)
|
|
)
|
|
|
|
blocking = [check for check in checks if not check.ok and check.fatal]
|
|
advisory = [check for check in checks if not check.ok and not check.fatal]
|
|
|
|
if args.json:
|
|
print(
|
|
json.dumps(
|
|
{
|
|
"product": PRODUCT_NAME,
|
|
"version": PRODUCT_VERSION,
|
|
"ready": not blocking,
|
|
"checks": [
|
|
{
|
|
"name": check.name,
|
|
"ok": check.ok,
|
|
"detail": check.detail,
|
|
"fatal": check.fatal,
|
|
}
|
|
for check in checks
|
|
],
|
|
},
|
|
indent=2,
|
|
)
|
|
)
|
|
else:
|
|
print(f"{PRODUCT_NAME} {PRODUCT_VERSION} — host preflight")
|
|
for check in checks:
|
|
mark = "OK " if check.ok else ("FAIL" if check.fatal else "WARN")
|
|
print(f" {mark} {check.name:24} {check.detail}")
|
|
print()
|
|
if blocking:
|
|
print(f"{len(blocking)} problem(s) must be fixed before installing.")
|
|
elif advisory:
|
|
print(f"Host is ready. {len(advisory)} advisory note(s) above.")
|
|
else:
|
|
print("Host is ready.")
|
|
return 1 if blocking else 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|