"""Build, scan and clean-install a ModelForge candidate in an isolated Compose project.""" from __future__ import annotations import argparse import atexit import base64 import json import os import secrets import shutil import subprocess import sys import time from datetime import UTC, datetime from pathlib import Path from urllib.parse import urlparse ROOT = Path(__file__).resolve().parents[1] IMAGES = ( "modelforge-api", "modelforge-web", "modelforge-node-agent", "modelforge-runtime-worker", ) CONFIG_SEED_IMAGE = ( "redis:7-alpine@sha256:ff02b58f971e7d7d156a1267e283fcbbeee91773b6aa36c49dac28ecfe28eadf" ) def run( *args: str, cwd: Path = ROOT, env: dict[str, str] | None = None, check: bool = True, capture: bool = False, ) -> str: completed = subprocess.run( list(args), cwd=cwd, env=env, text=True, encoding="utf-8", errors="replace", stdout=subprocess.PIPE if capture else None, stderr=subprocess.STDOUT if capture else None, check=False, ) output = (completed.stdout or "").strip() if check and completed.returncode: raise RuntimeError(f"{' '.join(args)} failed ({completed.returncode}): {output[-2000:]}") return output def container_http_status( container: str, url: str, *, operator: bool = False ) -> tuple[int, str]: probe = """ import json import os import sys import urllib.error import urllib.request headers = {} if sys.argv[2] == "operator": headers["X-ModelForge-Admin-Token"] = os.environ["MODELFORGE_OPERATOR_API_KEY"] request = urllib.request.Request(sys.argv[1], headers=headers) try: with urllib.request.urlopen(request, timeout=10) as response: status = response.status body = response.read().decode("utf-8", "replace") except urllib.error.HTTPError as error: status = error.code body = error.read().decode("utf-8", "replace") except OSError as error: status = 0 body = str(error) print(json.dumps({"status": status, "body": body})) """ output = run( "docker", "exec", container, "python", "-c", probe, url, "operator" if operator else "anonymous", capture=True, ) result = json.loads(output) return int(result["status"]), str(result["body"]) def wait_for_container_status( container: str, url: str, expected: int, timeout: int = 120 ) -> str: deadline = time.monotonic() + timeout last = "no response" while time.monotonic() < deadline: try: status, body = container_http_status(container, url) last = f"HTTP {status}: {body[:200]}" if status == expected: return body except OSError as exc: last = str(exc) time.sleep(2) raise RuntimeError(f"{url} did not reach HTTP {expected}: {last}") def provision_database_roles( compose: tuple[str, ...], env: dict[str, str], database: str ) -> None: """Apply the idempotent role bootstrap through the Docker API. Gitea Actions talks to a sibling Docker daemon. Host bind mounts therefore resolve on the daemon, not in the job container, so the normal init-directory mount is intentionally not relied on by acceptance. ``docker cp`` preserves the exact release SQL while working for both local and remote daemons. """ postgres = run(*compose, "ps", "--quiet", "postgres", env=env, capture=True) if not postgres: raise RuntimeError("acceptance PostgreSQL container was not created") bootstrap = ROOT / "deploy" / "postgres" / "init" / "001-modelforge-roles.sql" container_path = "/tmp/001-modelforge-roles.sql" run("docker", "cp", str(bootstrap), f"{postgres}:{container_path}") try: run( "docker", "exec", postgres, "psql", "--username", env.get("MODELFORGE_POSTGRES_ADMIN_USER", "postgres"), "--dbname", database, "--file", container_path, ) finally: run("docker", "exec", postgres, "rm", "--force", container_path, check=False) def provision_config_volume(project: str) -> None: """Populate the read-only API config volume through the Docker API.""" volume = f"{project}_acceptance-config" seed = f"{project}-config-seed" run( "docker", "volume", "create", "--label", f"com.docker.compose.project={project}", "--label", "com.docker.compose.volume=acceptance-config", volume, ) run( "docker", "create", "--name", seed, "--label", f"com.docker.compose.project={project}", "--volume", f"{volume}:/app/config", CONFIG_SEED_IMAGE, "true", ) try: run("docker", "cp", f"{ROOT / 'config'}/.", f"{seed}:/app/config") finally: run("docker", "container", "rm", "--force", seed, check=False) def cleanup_candidate_images(version: str, commit: str) -> None: """Remove only tags stamped by this exact acceptance commit.""" for image in IMAGES: tag = f"{image}:{version}" revision = run( "docker", "inspect", "--format", '{{index .Config.Labels "org.opencontainers.image.revision"}}', tag, check=False, capture=True, ) if revision == commit: print(f"Removing acceptance image {tag}", flush=True) run("docker", "image", "rm", "--force", tag, check=False) def main() -> int: parser = argparse.ArgumentParser() parser.add_argument("--public-api-origin", required=True) parser.add_argument("--output", type=Path, default=Path("acceptance-evidence")) parser.add_argument("--trivy", default="trivy") parser.add_argument("--project-suffix", default=os.environ.get("GITHUB_RUN_ID", "manual")) args = parser.parse_args() origin = args.public_api_origin.rstrip("/") parsed = urlparse(origin) if parsed.scheme not in {"http", "https"} or not parsed.netloc or parsed.path: raise SystemExit("--public-api-origin must be a bare absolute HTTP(S) origin") if shutil.which("docker") is None: raise SystemExit("Docker is required on the acceptance runner") if shutil.which(args.trivy) is None: raise SystemExit(f"Trivy executable not found: {args.trivy}") commit = run("git", "rev-parse", "HEAD", capture=True) if run("git", "status", "--porcelain", capture=True): raise SystemExit("Acceptance must run from a clean, exact source commit") version = (ROOT / "VERSION").read_text(encoding="utf-8").strip() atexit.register(cleanup_candidate_images, version, commit) built_at = datetime.now(UTC).isoformat() output = args.output.resolve() output.mkdir(parents=True, exist_ok=True) project_suffix = "".join(ch for ch in args.project_suffix.lower() if ch.isalnum())[-24:] project = f"modelforge-rc-{project_suffix or secrets.token_hex(6)}" if not project.startswith("modelforge-rc-"): raise SystemExit("refusing a non-RC Compose project name") run( sys.executable, str(ROOT / "scripts" / "release_build.py"), "--output", str(output / "release"), "--public-api-origin", origin, ) image_records: list[dict[str, str]] = [] for image in IMAGES: tag = f"{image}:{version}" image_id = run("docker", "inspect", "--format", "{{.Id}}", tag, capture=True) raw_report = output / f"trivy-{image}.json" trivy_summary = output / f"trivy-{image}-summary.json" run( args.trivy, "image", "--scanners", "vuln", "--severity", "UNKNOWN,LOW,MEDIUM,HIGH,CRITICAL", "--format", "json", "--output", str(raw_report), image_id, ) run( sys.executable, str(ROOT / "scripts" / "validate_trivy_report.py"), "--report", str(raw_report), "--image-id", image_id, "--summary", str(trivy_summary), "--reviewed-unfixed", str(ROOT / "config" / "public-candidate-unfixed-vulnerabilities.json"), ) image_records.append({"name": image, "tag": tag, "image_id": image_id}) admin_password = secrets.token_hex(24) owner_password = secrets.token_hex(24) runtime_password = secrets.token_hex(24) operator_key = secrets.token_urlsafe(48) backup_key = base64.urlsafe_b64encode(secrets.token_bytes(32)).decode("ascii") database = "modelforge_rc" env = os.environ.copy() env.update( { "COMPOSE_PROJECT_NAME": project, "MODELFORGE_VERSION": version, "MODELFORGE_COMMIT": commit, "MODELFORGE_BUILT_AT": built_at, "MODELFORGE_API_IMAGE": f"modelforge-api:{version}", "MODELFORGE_WEB_IMAGE": f"modelforge-web:{version}", "MODELFORGE_NODE_AGENT_IMAGE": f"modelforge-node-agent:{version}", "MODELFORGE_RUNTIME_WORKER_IMAGE": f"modelforge-runtime-worker:{version}", "MODELFORGE_POSTGRES_DB": database, "MODELFORGE_POSTGRES_ADMIN_PASSWORD": admin_password, "MODELFORGE_MIGRATION_DB_PASSWORD": owner_password, "MODELFORGE_RUNTIME_DB_PASSWORD": runtime_password, "MODELFORGE_MIGRATION_DATABASE_URL": ( f"postgresql+psycopg://modelforge:{owner_password}@postgres:5432/{database}" ), "MODELFORGE_RUNTIME_DATABASE_URL": ( f"postgresql+psycopg://modelforge_runtime:{runtime_password}@postgres:5432/{database}" ), "MODELFORGE_OPERATOR_API_KEY": operator_key, "MODELFORGE_BACKUP_ENCRYPTION_KEY": backup_key, "MODELFORGE_CORS_ORIGINS": origin, "VITE_API_BASE_URL": origin, "MODELFORGE_SOURCE_COMMIT": commit, "MODELFORGE_SOURCE_REFERENCE": "rc-acceptance", "MODELFORGE_SOURCE_REPOSITORY": "public-source-candidate", "MODELFORGE_API_BIND": "127.0.0.1", "MODELFORGE_API_PUBLISHED_PORT": "0", "MODELFORGE_WEB_BIND": "127.0.0.1", "MODELFORGE_WEB_PORT": "0", "MODELFORGE_POSTGRES_BIND": "127.0.0.1", "MODELFORGE_POSTGRES_PORT": "0", "MODELFORGE_REDIS_BIND": "127.0.0.1", "MODELFORGE_REDIS_PORT": "0", "MODELFORGE_ALLOW_REMOTE_CODE": "false", "MODELFORGE_RESTORE_ALLOW_PRODUCTION_TARGET": "false", } ) acceptance_override = output / "acceptance-compose.override.yml" acceptance_override.write_text( "services:\n" " api:\n" " volumes:\n" " - type: volume\n" " source: acceptance-config\n" " target: /app/config\n" " read_only: true\n" "volumes:\n" " acceptance-config:\n", encoding="utf-8", newline="\n", ) compose = ( "docker", "compose", "-p", project, "-f", str(ROOT / "docker-compose.yml"), "-f", str(ROOT / "docker-compose.production.yml"), "-f", str(acceptance_override), ) summary: dict[str, object] = { "schema_version": 1, "project": project, "source_commit": commit, "version": version, "public_api_origin": origin, "production_changed": False, "compute_identity_created": False, "images": image_records, "checks": {}, } try: run(*compose, "config", "-q", env=env) run(*compose, "up", "-d", "--no-build", "--wait", "postgres", "redis", env=env) provision_database_roles(compose, env, database) provision_config_volume(project) run(*compose, "up", "-d", "--no-build", "--wait", "api", "web", env=env) api_container = run(*compose, "ps", "--quiet", "api", env=env, capture=True) if not api_container: raise RuntimeError("acceptance API container was not created") api = "http://127.0.0.1:8000" wait_for_container_status(api_container, f"{api}/api/v1/health/live", 200) wait_for_container_status(api_container, f"{api}/api/v1/health/ready", 200) version_status, version_body = container_http_status( api_container, f"{api}/api/v1/version" ) unauthenticated, _ = container_http_status( api_container, f"{api}/api/v1/admin/recovery/dashboard" ) authenticated, _ = container_http_status( api_container, f"{api}/api/v1/admin/recovery/dashboard", operator=True, ) wait_for_container_status(api_container, "http://web:3000/", 200) if version_status != 200 or unauthenticated != 401 or authenticated != 200: raise RuntimeError( "acceptance boundary mismatch: " f"version={version_status}, unauthenticated={unauthenticated}, " f"authenticated={authenticated}" ) version_payload = json.loads(version_body) if version_payload.get("version") != version: raise RuntimeError(f"running version does not match {version}: {version_payload}") volumes = run( "docker", "volume", "ls", "--filter", f"label=com.docker.compose.project={project}", "--format", "{{.Name}}", capture=True, ).splitlines() if not volumes or any(not volume.startswith(f"{project}_") for volume in volumes): raise RuntimeError(f"Compose volumes are not isolated under {project}: {volumes}") summary["checks"] = { "live": 200, "ready": 200, "version": version_status, "admin_without_key": unauthenticated, "admin_with_key": authenticated, "console": 200, "database_roles_provisioned": True, "network_probe_container": "api", "isolated_volumes": sorted(volumes), } summary["result"] = "PASS" (output / "compose-ps.json").write_text( run(*compose, "ps", "--format", "json", env=env, capture=True) + "\n", encoding="utf-8", newline="\n", ) except Exception: summary["result"] = "FAIL" (output / "compose-logs.txt").write_text( run(*compose, "logs", "--no-color", env=env, check=False, capture=True) + "\n", encoding="utf-8", newline="\n", ) raise finally: run(*compose, "down", "--volumes", "--remove-orphans", env=env, check=False) (output / "acceptance-summary.json").write_text( json.dumps(summary, indent=2, sort_keys=True) + "\n", encoding="utf-8", newline="\n", ) print(f"Isolated RC acceptance PASS for {commit} in {project}") return 0 if __name__ == "__main__": raise SystemExit(main())