Initial public ModelForge release
This commit is contained in:
Jens
@@ -0,0 +1,442 @@
|
||||
"""Build, scan and clean-install a ModelForge candidate in an isolated Compose project."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import atexit
|
||||
import base64
|
||||
import json
|
||||
import os
|
||||
import secrets
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from urllib.parse import urlparse
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
IMAGES = (
|
||||
"modelforge-api",
|
||||
"modelforge-web",
|
||||
"modelforge-node-agent",
|
||||
"modelforge-runtime-worker",
|
||||
)
|
||||
CONFIG_SEED_IMAGE = (
|
||||
"redis:7-alpine@sha256:ff02b58f971e7d7d156a1267e283fcbbeee91773b6aa36c49dac28ecfe28eadf"
|
||||
)
|
||||
|
||||
|
||||
def run(
|
||||
*args: str,
|
||||
cwd: Path = ROOT,
|
||||
env: dict[str, str] | None = None,
|
||||
check: bool = True,
|
||||
capture: bool = False,
|
||||
) -> str:
|
||||
completed = subprocess.run(
|
||||
list(args),
|
||||
cwd=cwd,
|
||||
env=env,
|
||||
text=True,
|
||||
encoding="utf-8",
|
||||
errors="replace",
|
||||
stdout=subprocess.PIPE if capture else None,
|
||||
stderr=subprocess.STDOUT if capture else None,
|
||||
check=False,
|
||||
)
|
||||
output = (completed.stdout or "").strip()
|
||||
if check and completed.returncode:
|
||||
raise RuntimeError(f"{' '.join(args)} failed ({completed.returncode}): {output[-2000:]}")
|
||||
return output
|
||||
|
||||
|
||||
def container_http_status(
|
||||
container: str, url: str, *, operator: bool = False
|
||||
) -> tuple[int, str]:
|
||||
probe = """
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
|
||||
headers = {}
|
||||
if sys.argv[2] == "operator":
|
||||
headers["X-ModelForge-Admin-Token"] = os.environ["MODELFORGE_OPERATOR_API_KEY"]
|
||||
request = urllib.request.Request(sys.argv[1], headers=headers)
|
||||
try:
|
||||
with urllib.request.urlopen(request, timeout=10) as response:
|
||||
status = response.status
|
||||
body = response.read().decode("utf-8", "replace")
|
||||
except urllib.error.HTTPError as error:
|
||||
status = error.code
|
||||
body = error.read().decode("utf-8", "replace")
|
||||
except OSError as error:
|
||||
status = 0
|
||||
body = str(error)
|
||||
print(json.dumps({"status": status, "body": body}))
|
||||
"""
|
||||
output = run(
|
||||
"docker",
|
||||
"exec",
|
||||
container,
|
||||
"python",
|
||||
"-c",
|
||||
probe,
|
||||
url,
|
||||
"operator" if operator else "anonymous",
|
||||
capture=True,
|
||||
)
|
||||
result = json.loads(output)
|
||||
return int(result["status"]), str(result["body"])
|
||||
|
||||
|
||||
def wait_for_container_status(
|
||||
container: str, url: str, expected: int, timeout: int = 120
|
||||
) -> str:
|
||||
deadline = time.monotonic() + timeout
|
||||
last = "no response"
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
status, body = container_http_status(container, url)
|
||||
last = f"HTTP {status}: {body[:200]}"
|
||||
if status == expected:
|
||||
return body
|
||||
except OSError as exc:
|
||||
last = str(exc)
|
||||
time.sleep(2)
|
||||
raise RuntimeError(f"{url} did not reach HTTP {expected}: {last}")
|
||||
|
||||
|
||||
def provision_database_roles(
|
||||
compose: tuple[str, ...], env: dict[str, str], database: str
|
||||
) -> None:
|
||||
"""Apply the idempotent role bootstrap through the Docker API.
|
||||
|
||||
Gitea Actions talks to a sibling Docker daemon. Host bind mounts therefore resolve on the
|
||||
daemon, not in the job container, so the normal init-directory mount is intentionally not
|
||||
relied on by acceptance. ``docker cp`` preserves the exact release SQL while working for both
|
||||
local and remote daemons.
|
||||
"""
|
||||
|
||||
postgres = run(*compose, "ps", "--quiet", "postgres", env=env, capture=True)
|
||||
if not postgres:
|
||||
raise RuntimeError("acceptance PostgreSQL container was not created")
|
||||
bootstrap = ROOT / "deploy" / "postgres" / "init" / "001-modelforge-roles.sql"
|
||||
container_path = "/tmp/001-modelforge-roles.sql"
|
||||
run("docker", "cp", str(bootstrap), f"{postgres}:{container_path}")
|
||||
try:
|
||||
run(
|
||||
"docker",
|
||||
"exec",
|
||||
postgres,
|
||||
"psql",
|
||||
"--username",
|
||||
env.get("MODELFORGE_POSTGRES_ADMIN_USER", "postgres"),
|
||||
"--dbname",
|
||||
database,
|
||||
"--file",
|
||||
container_path,
|
||||
)
|
||||
finally:
|
||||
run("docker", "exec", postgres, "rm", "--force", container_path, check=False)
|
||||
|
||||
|
||||
def provision_config_volume(project: str) -> None:
|
||||
"""Populate the read-only API config volume through the Docker API."""
|
||||
|
||||
volume = f"{project}_acceptance-config"
|
||||
seed = f"{project}-config-seed"
|
||||
run(
|
||||
"docker",
|
||||
"volume",
|
||||
"create",
|
||||
"--label",
|
||||
f"com.docker.compose.project={project}",
|
||||
"--label",
|
||||
"com.docker.compose.volume=acceptance-config",
|
||||
volume,
|
||||
)
|
||||
run(
|
||||
"docker",
|
||||
"create",
|
||||
"--name",
|
||||
seed,
|
||||
"--label",
|
||||
f"com.docker.compose.project={project}",
|
||||
"--volume",
|
||||
f"{volume}:/app/config",
|
||||
CONFIG_SEED_IMAGE,
|
||||
"true",
|
||||
)
|
||||
try:
|
||||
run("docker", "cp", f"{ROOT / 'config'}/.", f"{seed}:/app/config")
|
||||
finally:
|
||||
run("docker", "container", "rm", "--force", seed, check=False)
|
||||
|
||||
|
||||
def cleanup_candidate_images(version: str, commit: str) -> None:
|
||||
"""Remove only tags stamped by this exact acceptance commit."""
|
||||
|
||||
for image in IMAGES:
|
||||
tag = f"{image}:{version}"
|
||||
revision = run(
|
||||
"docker",
|
||||
"inspect",
|
||||
"--format",
|
||||
'{{index .Config.Labels "org.opencontainers.image.revision"}}',
|
||||
tag,
|
||||
check=False,
|
||||
capture=True,
|
||||
)
|
||||
if revision == commit:
|
||||
print(f"Removing acceptance image {tag}", flush=True)
|
||||
run("docker", "image", "rm", "--force", tag, check=False)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--public-api-origin", required=True)
|
||||
parser.add_argument("--output", type=Path, default=Path("acceptance-evidence"))
|
||||
parser.add_argument("--trivy", default="trivy")
|
||||
parser.add_argument("--project-suffix", default=os.environ.get("GITHUB_RUN_ID", "manual"))
|
||||
args = parser.parse_args()
|
||||
|
||||
origin = args.public_api_origin.rstrip("/")
|
||||
parsed = urlparse(origin)
|
||||
if parsed.scheme not in {"http", "https"} or not parsed.netloc or parsed.path:
|
||||
raise SystemExit("--public-api-origin must be a bare absolute HTTP(S) origin")
|
||||
if shutil.which("docker") is None:
|
||||
raise SystemExit("Docker is required on the acceptance runner")
|
||||
if shutil.which(args.trivy) is None:
|
||||
raise SystemExit(f"Trivy executable not found: {args.trivy}")
|
||||
|
||||
commit = run("git", "rev-parse", "HEAD", capture=True)
|
||||
if run("git", "status", "--porcelain", capture=True):
|
||||
raise SystemExit("Acceptance must run from a clean, exact source commit")
|
||||
version = (ROOT / "VERSION").read_text(encoding="utf-8").strip()
|
||||
atexit.register(cleanup_candidate_images, version, commit)
|
||||
built_at = datetime.now(UTC).isoformat()
|
||||
output = args.output.resolve()
|
||||
output.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
project_suffix = "".join(ch for ch in args.project_suffix.lower() if ch.isalnum())[-24:]
|
||||
project = f"modelforge-rc-{project_suffix or secrets.token_hex(6)}"
|
||||
if not project.startswith("modelforge-rc-"):
|
||||
raise SystemExit("refusing a non-RC Compose project name")
|
||||
|
||||
run(
|
||||
sys.executable,
|
||||
str(ROOT / "scripts" / "release_build.py"),
|
||||
"--output",
|
||||
str(output / "release"),
|
||||
"--public-api-origin",
|
||||
origin,
|
||||
)
|
||||
|
||||
image_records: list[dict[str, str]] = []
|
||||
for image in IMAGES:
|
||||
tag = f"{image}:{version}"
|
||||
image_id = run("docker", "inspect", "--format", "{{.Id}}", tag, capture=True)
|
||||
raw_report = output / f"trivy-{image}.json"
|
||||
trivy_summary = output / f"trivy-{image}-summary.json"
|
||||
run(
|
||||
args.trivy,
|
||||
"image",
|
||||
"--scanners",
|
||||
"vuln",
|
||||
"--severity",
|
||||
"UNKNOWN,LOW,MEDIUM,HIGH,CRITICAL",
|
||||
"--format",
|
||||
"json",
|
||||
"--output",
|
||||
str(raw_report),
|
||||
image_id,
|
||||
)
|
||||
run(
|
||||
sys.executable,
|
||||
str(ROOT / "scripts" / "validate_trivy_report.py"),
|
||||
"--report",
|
||||
str(raw_report),
|
||||
"--image-id",
|
||||
image_id,
|
||||
"--summary",
|
||||
str(trivy_summary),
|
||||
"--reviewed-unfixed",
|
||||
str(ROOT / "config" / "public-candidate-unfixed-vulnerabilities.json"),
|
||||
)
|
||||
image_records.append({"name": image, "tag": tag, "image_id": image_id})
|
||||
|
||||
admin_password = secrets.token_hex(24)
|
||||
owner_password = secrets.token_hex(24)
|
||||
runtime_password = secrets.token_hex(24)
|
||||
operator_key = secrets.token_urlsafe(48)
|
||||
backup_key = base64.urlsafe_b64encode(secrets.token_bytes(32)).decode("ascii")
|
||||
database = "modelforge_rc"
|
||||
env = os.environ.copy()
|
||||
env.update(
|
||||
{
|
||||
"COMPOSE_PROJECT_NAME": project,
|
||||
"MODELFORGE_VERSION": version,
|
||||
"MODELFORGE_COMMIT": commit,
|
||||
"MODELFORGE_BUILT_AT": built_at,
|
||||
"MODELFORGE_API_IMAGE": f"modelforge-api:{version}",
|
||||
"MODELFORGE_WEB_IMAGE": f"modelforge-web:{version}",
|
||||
"MODELFORGE_NODE_AGENT_IMAGE": f"modelforge-node-agent:{version}",
|
||||
"MODELFORGE_RUNTIME_WORKER_IMAGE": f"modelforge-runtime-worker:{version}",
|
||||
"MODELFORGE_POSTGRES_DB": database,
|
||||
"MODELFORGE_POSTGRES_ADMIN_PASSWORD": admin_password,
|
||||
"MODELFORGE_MIGRATION_DB_PASSWORD": owner_password,
|
||||
"MODELFORGE_RUNTIME_DB_PASSWORD": runtime_password,
|
||||
"MODELFORGE_MIGRATION_DATABASE_URL": (
|
||||
f"postgresql+psycopg://modelforge:{owner_password}@postgres:5432/{database}"
|
||||
),
|
||||
"MODELFORGE_RUNTIME_DATABASE_URL": (
|
||||
f"postgresql+psycopg://modelforge_runtime:{runtime_password}@postgres:5432/{database}"
|
||||
),
|
||||
"MODELFORGE_OPERATOR_API_KEY": operator_key,
|
||||
"MODELFORGE_BACKUP_ENCRYPTION_KEY": backup_key,
|
||||
"MODELFORGE_CORS_ORIGINS": origin,
|
||||
"VITE_API_BASE_URL": origin,
|
||||
"MODELFORGE_SOURCE_COMMIT": commit,
|
||||
"MODELFORGE_SOURCE_REFERENCE": "rc-acceptance",
|
||||
"MODELFORGE_SOURCE_REPOSITORY": "public-source-candidate",
|
||||
"MODELFORGE_API_BIND": "127.0.0.1",
|
||||
"MODELFORGE_API_PUBLISHED_PORT": "0",
|
||||
"MODELFORGE_WEB_BIND": "127.0.0.1",
|
||||
"MODELFORGE_WEB_PORT": "0",
|
||||
"MODELFORGE_POSTGRES_BIND": "127.0.0.1",
|
||||
"MODELFORGE_POSTGRES_PORT": "0",
|
||||
"MODELFORGE_REDIS_BIND": "127.0.0.1",
|
||||
"MODELFORGE_REDIS_PORT": "0",
|
||||
"MODELFORGE_ALLOW_REMOTE_CODE": "false",
|
||||
"MODELFORGE_RESTORE_ALLOW_PRODUCTION_TARGET": "false",
|
||||
}
|
||||
)
|
||||
acceptance_override = output / "acceptance-compose.override.yml"
|
||||
acceptance_override.write_text(
|
||||
"services:\n"
|
||||
" api:\n"
|
||||
" volumes:\n"
|
||||
" - type: volume\n"
|
||||
" source: acceptance-config\n"
|
||||
" target: /app/config\n"
|
||||
" read_only: true\n"
|
||||
"volumes:\n"
|
||||
" acceptance-config:\n",
|
||||
encoding="utf-8",
|
||||
newline="\n",
|
||||
)
|
||||
compose = (
|
||||
"docker",
|
||||
"compose",
|
||||
"-p",
|
||||
project,
|
||||
"-f",
|
||||
str(ROOT / "docker-compose.yml"),
|
||||
"-f",
|
||||
str(ROOT / "docker-compose.production.yml"),
|
||||
"-f",
|
||||
str(acceptance_override),
|
||||
)
|
||||
|
||||
summary: dict[str, object] = {
|
||||
"schema_version": 1,
|
||||
"project": project,
|
||||
"source_commit": commit,
|
||||
"version": version,
|
||||
"public_api_origin": origin,
|
||||
"production_changed": False,
|
||||
"compute_identity_created": False,
|
||||
"images": image_records,
|
||||
"checks": {},
|
||||
}
|
||||
try:
|
||||
run(*compose, "config", "-q", env=env)
|
||||
run(*compose, "up", "-d", "--no-build", "--wait", "postgres", "redis", env=env)
|
||||
provision_database_roles(compose, env, database)
|
||||
provision_config_volume(project)
|
||||
run(*compose, "up", "-d", "--no-build", "--wait", "api", "web", env=env)
|
||||
api_container = run(*compose, "ps", "--quiet", "api", env=env, capture=True)
|
||||
if not api_container:
|
||||
raise RuntimeError("acceptance API container was not created")
|
||||
api = "http://127.0.0.1:8000"
|
||||
|
||||
wait_for_container_status(api_container, f"{api}/api/v1/health/live", 200)
|
||||
wait_for_container_status(api_container, f"{api}/api/v1/health/ready", 200)
|
||||
version_status, version_body = container_http_status(
|
||||
api_container, f"{api}/api/v1/version"
|
||||
)
|
||||
unauthenticated, _ = container_http_status(
|
||||
api_container, f"{api}/api/v1/admin/recovery/dashboard"
|
||||
)
|
||||
authenticated, _ = container_http_status(
|
||||
api_container,
|
||||
f"{api}/api/v1/admin/recovery/dashboard",
|
||||
operator=True,
|
||||
)
|
||||
wait_for_container_status(api_container, "http://web:3000/", 200)
|
||||
if version_status != 200 or unauthenticated != 401 or authenticated != 200:
|
||||
raise RuntimeError(
|
||||
"acceptance boundary mismatch: "
|
||||
f"version={version_status}, unauthenticated={unauthenticated}, "
|
||||
f"authenticated={authenticated}"
|
||||
)
|
||||
version_payload = json.loads(version_body)
|
||||
if version_payload.get("version") != version:
|
||||
raise RuntimeError(f"running version does not match {version}: {version_payload}")
|
||||
|
||||
volumes = run(
|
||||
"docker",
|
||||
"volume",
|
||||
"ls",
|
||||
"--filter",
|
||||
f"label=com.docker.compose.project={project}",
|
||||
"--format",
|
||||
"{{.Name}}",
|
||||
capture=True,
|
||||
).splitlines()
|
||||
if not volumes or any(not volume.startswith(f"{project}_") for volume in volumes):
|
||||
raise RuntimeError(f"Compose volumes are not isolated under {project}: {volumes}")
|
||||
summary["checks"] = {
|
||||
"live": 200,
|
||||
"ready": 200,
|
||||
"version": version_status,
|
||||
"admin_without_key": unauthenticated,
|
||||
"admin_with_key": authenticated,
|
||||
"console": 200,
|
||||
"database_roles_provisioned": True,
|
||||
"network_probe_container": "api",
|
||||
"isolated_volumes": sorted(volumes),
|
||||
}
|
||||
summary["result"] = "PASS"
|
||||
(output / "compose-ps.json").write_text(
|
||||
run(*compose, "ps", "--format", "json", env=env, capture=True) + "\n",
|
||||
encoding="utf-8",
|
||||
newline="\n",
|
||||
)
|
||||
except Exception:
|
||||
summary["result"] = "FAIL"
|
||||
(output / "compose-logs.txt").write_text(
|
||||
run(*compose, "logs", "--no-color", env=env, check=False, capture=True) + "\n",
|
||||
encoding="utf-8",
|
||||
newline="\n",
|
||||
)
|
||||
raise
|
||||
finally:
|
||||
run(*compose, "down", "--volumes", "--remove-orphans", env=env, check=False)
|
||||
(output / "acceptance-summary.json").write_text(
|
||||
json.dumps(summary, indent=2, sort_keys=True) + "\n",
|
||||
encoding="utf-8",
|
||||
newline="\n",
|
||||
)
|
||||
|
||||
print(f"Isolated RC acceptance PASS for {commit} in {project}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in New Issue
Block a user