Initial public ModelForge release
This commit is contained in:
Jens
@@ -0,0 +1,282 @@
|
||||
"""Check a host before installing ModelForge, and say exactly what is wrong.
|
||||
|
||||
Run this first on a clean machine. It answers the question an operator actually has — "will this
|
||||
work here?" — before anything is downloaded, built or started, and it never changes the system it
|
||||
is inspecting.
|
||||
|
||||
python scripts/preflight.py
|
||||
python scripts/preflight.py --production # also check the production configuration rules
|
||||
|
||||
Exit status is 0 when the host can run ModelForge, 1 when something must be fixed.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import socket
|
||||
import subprocess # noqa: S404 - fixed argv, never a shell
|
||||
import sys
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "backend" / "src"))
|
||||
|
||||
from modelforge_api.domain.release import ( # noqa: E402
|
||||
MINIMUM_POSTGRES_MAJOR,
|
||||
PRODUCT_NAME,
|
||||
PRODUCT_VERSION,
|
||||
)
|
||||
|
||||
MINIMUM_DOCKER_MAJOR = 24
|
||||
MINIMUM_COMPOSE_MAJOR = 2
|
||||
RECOMMENDED_FREE_BYTES = 50 * 1024**3
|
||||
|
||||
#: Ports the deployment publishes, and the variable that moves each one.
|
||||
PUBLISHED_PORTS = {
|
||||
"MODELFORGE_API_PUBLISHED_PORT": 8000,
|
||||
"MODELFORGE_WEB_PORT": 3000,
|
||||
"MODELFORGE_POSTGRES_PORT": 5432,
|
||||
"MODELFORGE_REDIS_PORT": 6379,
|
||||
}
|
||||
|
||||
|
||||
@dataclass
|
||||
class Check:
|
||||
name: str
|
||||
ok: bool
|
||||
detail: str
|
||||
fatal: bool = True
|
||||
|
||||
|
||||
def run(*args: str) -> tuple[int, str]:
|
||||
try:
|
||||
completed = subprocess.run( # noqa: S603 - fixed argv, no shell
|
||||
list(args),
|
||||
capture_output=True,
|
||||
text=True,
|
||||
encoding="utf-8",
|
||||
errors="replace",
|
||||
timeout=60,
|
||||
check=False,
|
||||
)
|
||||
except (OSError, subprocess.TimeoutExpired) as error:
|
||||
return 1, f"{type(error).__name__}"
|
||||
return completed.returncode, (completed.stdout or completed.stderr or "").strip()
|
||||
|
||||
|
||||
def _major(value: str) -> int:
|
||||
digits = ""
|
||||
for character in value.lstrip("v"):
|
||||
if character.isdigit():
|
||||
digits += character
|
||||
else:
|
||||
break
|
||||
return int(digits) if digits else 0
|
||||
|
||||
|
||||
def check_docker() -> list[Check]:
|
||||
checks: list[Check] = []
|
||||
if shutil.which("docker") is None:
|
||||
return [Check("docker", False, "docker is not on PATH")]
|
||||
code, version = run("docker", "version", "--format", "{{.Server.Version}}")
|
||||
if code != 0:
|
||||
return [Check("docker", False, f"the Docker daemon is not reachable: {version[:120]}")]
|
||||
major = _major(version)
|
||||
checks.append(
|
||||
Check(
|
||||
"docker engine",
|
||||
major >= MINIMUM_DOCKER_MAJOR,
|
||||
f"{version} (minimum {MINIMUM_DOCKER_MAJOR})",
|
||||
)
|
||||
)
|
||||
code, compose = run("docker", "compose", "version", "--short")
|
||||
checks.append(
|
||||
Check(
|
||||
"docker compose",
|
||||
code == 0 and _major(compose) >= MINIMUM_COMPOSE_MAJOR,
|
||||
f"{compose or 'absent'} (minimum {MINIMUM_COMPOSE_MAJOR}.x)",
|
||||
)
|
||||
)
|
||||
return checks
|
||||
|
||||
|
||||
def check_gpu() -> list[Check]:
|
||||
"""A GPU is required on a compute node, not on the control-plane host."""
|
||||
|
||||
if shutil.which("nvidia-smi") is None:
|
||||
return [
|
||||
Check(
|
||||
"nvidia driver",
|
||||
True,
|
||||
"nvidia-smi is absent; required only on a GPU compute node",
|
||||
fatal=False,
|
||||
)
|
||||
]
|
||||
code, output = run("nvidia-smi", "--query-gpu=name,driver_version", "--format=csv,noheader")
|
||||
if code != 0:
|
||||
return [Check("nvidia driver", False, "nvidia-smi failed", fatal=False)]
|
||||
return [Check("nvidia driver", True, output.replace("\n", "; "), fatal=False)]
|
||||
|
||||
|
||||
def check_ports() -> list[Check]:
|
||||
checks = []
|
||||
for variable, default in PUBLISHED_PORTS.items():
|
||||
port = int(os.environ.get(variable, default))
|
||||
connection = socket.socket()
|
||||
connection.settimeout(1)
|
||||
try:
|
||||
connection.connect(("127.0.0.1", port))
|
||||
occupied = True
|
||||
except OSError:
|
||||
occupied = False
|
||||
finally:
|
||||
connection.close()
|
||||
checks.append(
|
||||
Check(
|
||||
f"port {port}",
|
||||
not occupied,
|
||||
f"already in use; move it with {variable}" if occupied else "free",
|
||||
fatal=False,
|
||||
)
|
||||
)
|
||||
return checks
|
||||
|
||||
|
||||
def check_storage(path: Path) -> list[Check]:
|
||||
try:
|
||||
usage = shutil.disk_usage(path)
|
||||
except OSError as error:
|
||||
return [Check("storage", False, f"{path} is not readable: {error}")]
|
||||
free_gib = usage.free / 1024**3
|
||||
return [
|
||||
Check(
|
||||
"free storage",
|
||||
usage.free >= RECOMMENDED_FREE_BYTES,
|
||||
f"{free_gib:.1f} GiB free at {path} "
|
||||
f"(recommended {RECOMMENDED_FREE_BYTES / 1024**3:.0f} GiB for model artifacts)",
|
||||
fatal=False,
|
||||
)
|
||||
]
|
||||
|
||||
|
||||
def check_configuration(production: bool) -> list[Check]:
|
||||
env_file = ROOT / ".env"
|
||||
example = ROOT / ".env.example"
|
||||
checks = [
|
||||
Check(".env.example", example.is_file(), "present" if example.is_file() else "missing")
|
||||
]
|
||||
if not env_file.is_file():
|
||||
checks.append(
|
||||
Check(
|
||||
".env",
|
||||
not production,
|
||||
"absent; copy .env.example to .env and fill in the generated secrets",
|
||||
fatal=production,
|
||||
)
|
||||
)
|
||||
return checks
|
||||
checks.append(Check(".env", True, "present"))
|
||||
if production:
|
||||
from modelforge_api.services.startup_validation import (
|
||||
StartupFailureCode,
|
||||
validate_settings,
|
||||
)
|
||||
from modelforge_api.settings import Settings
|
||||
|
||||
settings = Settings(_env_file=str(env_file)) # type: ignore[call-arg]
|
||||
report = validate_settings(settings)
|
||||
# Storage roots are container paths. Validating them against the host is meaningless — the
|
||||
# host has no /data/artifacts and is not supposed to — so they are checked inside the
|
||||
# container at startup, which is the only place the answer means anything.
|
||||
relevant = [
|
||||
problem
|
||||
for problem in report.problems
|
||||
if problem.code is not StartupFailureCode.INVALID_STORAGE_ROOT
|
||||
]
|
||||
for problem in relevant:
|
||||
checks.append(Check(problem.setting, False, f"{problem.code}: {problem.message}"))
|
||||
if not relevant:
|
||||
checks.append(
|
||||
Check(
|
||||
"production configuration",
|
||||
True,
|
||||
"no problems found; storage roots are validated inside the container",
|
||||
)
|
||||
)
|
||||
return checks
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--production", action="store_true")
|
||||
parser.add_argument("--storage-path", default=str(ROOT))
|
||||
parser.add_argument("--json", action="store_true")
|
||||
args = parser.parse_args(argv)
|
||||
|
||||
checks: list[Check] = []
|
||||
checks.append(
|
||||
Check(
|
||||
"python",
|
||||
sys.version_info >= (3, 12),
|
||||
f"{sys.version.split()[0]} (minimum 3.12)",
|
||||
fatal=False,
|
||||
)
|
||||
)
|
||||
checks += check_docker()
|
||||
checks += check_gpu()
|
||||
checks += check_ports()
|
||||
checks += check_storage(Path(args.storage_path))
|
||||
checks += check_configuration(args.production)
|
||||
checks.append(
|
||||
Check(
|
||||
"postgresql",
|
||||
True,
|
||||
f"provided by the compose deployment; minimum major {MINIMUM_POSTGRES_MAJOR}",
|
||||
fatal=False,
|
||||
)
|
||||
)
|
||||
|
||||
blocking = [check for check in checks if not check.ok and check.fatal]
|
||||
advisory = [check for check in checks if not check.ok and not check.fatal]
|
||||
|
||||
if args.json:
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"product": PRODUCT_NAME,
|
||||
"version": PRODUCT_VERSION,
|
||||
"ready": not blocking,
|
||||
"checks": [
|
||||
{
|
||||
"name": check.name,
|
||||
"ok": check.ok,
|
||||
"detail": check.detail,
|
||||
"fatal": check.fatal,
|
||||
}
|
||||
for check in checks
|
||||
],
|
||||
},
|
||||
indent=2,
|
||||
)
|
||||
)
|
||||
else:
|
||||
print(f"{PRODUCT_NAME} {PRODUCT_VERSION} — host preflight")
|
||||
for check in checks:
|
||||
mark = "OK " if check.ok else ("FAIL" if check.fatal else "WARN")
|
||||
print(f" {mark} {check.name:24} {check.detail}")
|
||||
print()
|
||||
if blocking:
|
||||
print(f"{len(blocking)} problem(s) must be fixed before installing.")
|
||||
elif advisory:
|
||||
print(f"Host is ready. {len(advisory)} advisory note(s) above.")
|
||||
else:
|
||||
print("Host is ready.")
|
||||
return 1 if blocking else 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in New Issue
Block a user