"""Packaging and configuration guarantees for the v1 release. The release artefact is what an operator receives. These tests hold it to the things that are only noticeable once it is too late: a secret that shipped, a production overlay that quietly accepts a development default, an image without the identity to trace it back to a commit. """ from __future__ import annotations import json import re import subprocess import sys import tomllib from pathlib import Path, PurePosixPath import pytest from modelforge_api.domain.configuration_reference import ( DEPLOYMENT_DOCS, SETTING_DOCS, Sensitivity, ) from modelforge_api.domain.release import PRODUCT_VERSION from modelforge_api.settings import Settings ROOT = Path(__file__).resolve().parents[2] DOCKERFILES = ( "backend/Dockerfile", "frontend/Dockerfile", "node-agent/Dockerfile", "runtime-worker/Dockerfile", ) # --------------------------------------------------------------------------- configuration def test_every_setting_is_documented() -> None: """A setting added without documentation fails the build rather than shipping quietly.""" undocumented = sorted(set(Settings.model_fields) - set(SETTING_DOCS)) assert not undocumented, f"undocumented settings: {undocumented}" def test_the_reference_documents_no_setting_that_no_longer_exists() -> None: stale = sorted(set(SETTING_DOCS) - set(Settings.model_fields)) assert not stale, f"documented settings that no longer exist: {stale}" def test_the_generated_configuration_files_are_current() -> None: """`.env.example` and the configuration reference are generated, never hand-edited.""" completed = subprocess.run( # noqa: S603 - fixed argv, no shell [sys.executable, str(ROOT / "scripts" / "generate_configuration_docs.py"), "--check"], capture_output=True, text=True, encoding="utf-8", errors="replace", cwd=ROOT, check=False, ) assert completed.returncode == 0, ( "configuration documentation is stale; run " f"scripts/generate_configuration_docs.py\n{completed.stdout}{completed.stderr}" ) def test_the_example_configuration_carries_no_secret_values() -> None: """The example is committed, so a real value in it would be published with the release.""" text = (ROOT / ".env.example").read_text("utf-8") secret_names = [ f"MODELFORGE_{name.upper()}" for name, doc in SETTING_DOCS.items() if doc.sensitivity is Sensitivity.SECRET ] for name in secret_names: for line in text.splitlines(): if line.startswith(f"{name}="): assert line == f"{name}=", f"{name} carries a value in .env.example" def test_every_required_production_setting_appears_in_the_example() -> None: text = (ROOT / ".env.example").read_text("utf-8") for name, doc in SETTING_DOCS.items(): if doc.required_in_production: assert f"MODELFORGE_{name.upper()}=" in text, f"{name} is required but not offered" def test_deployment_variables_are_documented_too() -> None: """Variables Compose and the agents read are still variables an operator has to set.""" text = (ROOT / ".env.example").read_text("utf-8") for name in DEPLOYMENT_DOCS: assert f"{name}=" in text, f"{name} is undocumented in .env.example" # --------------------------------------------------------------------------- production overlay def test_the_production_overlay_exists_and_sets_the_production_profile() -> None: """Without MODELFORGE_ENV=production every fail-closed startup rule stays switched off.""" text = (ROOT / "docker-compose.production.yml").read_text("utf-8") assert "MODELFORGE_ENV: production" in text @pytest.mark.parametrize( "variable", [ "MODELFORGE_POSTGRES_ADMIN_PASSWORD", "MODELFORGE_MIGRATION_DB_PASSWORD", "MODELFORGE_RUNTIME_DB_PASSWORD", "MODELFORGE_MIGRATION_DATABASE_URL", "MODELFORGE_RUNTIME_DATABASE_URL", "MODELFORGE_OPERATOR_API_KEY", "MODELFORGE_BACKUP_ENCRYPTION_KEY", "MODELFORGE_CORS_ORIGINS", ], ) def test_the_production_overlay_refuses_to_render_without_its_secrets(variable: str) -> None: """`${VAR:?message}` makes Compose fail before a single container starts.""" text = (ROOT / "docker-compose.production.yml").read_text("utf-8") assert re.search(rf"\$\{{{variable}:\?[^}}]+\}}", text), ( f"{variable} must use the ${{VAR:?message}} form so a missing value fails the render" ) def test_the_production_overlay_never_permits_remote_code() -> None: text = (ROOT / "docker-compose.production.yml").read_text("utf-8") assert 'MODELFORGE_ALLOW_REMOTE_CODE: "false"' in text def test_the_base_compose_file_is_not_mistakable_for_production() -> None: """The base file is development-only; role secrets are still required and never embedded.""" text = (ROOT / "docker-compose.yml").read_text("utf-8") assert "MODELFORGE_ENV: production" not in text # --------------------------------------------------------------------------- images @pytest.mark.parametrize("dockerfile", DOCKERFILES) def test_every_image_carries_oci_identity_labels(dockerfile: str) -> None: text = (ROOT / dockerfile).read_text("utf-8") for label in ( "org.opencontainers.image.version", "org.opencontainers.image.revision", "org.opencontainers.image.created", "org.opencontainers.image.source", ): assert label in text, f"{dockerfile} does not declare {label}" @pytest.mark.parametrize("dockerfile", DOCKERFILES) def test_every_image_accepts_build_identity_arguments(dockerfile: str) -> None: text = (ROOT / dockerfile).read_text("utf-8") for argument in ("MODELFORGE_VERSION", "MODELFORGE_COMMIT", "MODELFORGE_BUILT_AT"): assert f"ARG {argument}" in text, f"{dockerfile} does not accept {argument}" @pytest.mark.parametrize("dockerfile", DOCKERFILES) def test_every_image_declares_the_product_license(dockerfile: str) -> None: text = (ROOT / dockerfile).read_text("utf-8") assert 'org.opencontainers.image.licenses="AGPL-3.0-or-later"' in text assert 'org.opencontainers.image.licenses="Proprietary"' not in text def test_every_component_declares_the_product_license() -> None: for relative_path in ( "backend/pyproject.toml", "node-agent/pyproject.toml", "runtime-worker/pyproject.toml", ): metadata = tomllib.loads((ROOT / relative_path).read_text("utf-8")) assert metadata["project"]["license"] == "AGPL-3.0-or-later" package = json.loads((ROOT / "frontend/package.json").read_text("utf-8")) lock = json.loads((ROOT / "frontend/package-lock.json").read_text("utf-8")) assert package["license"] == "AGPL-3.0-or-later" assert lock["packages"][""]["license"] == "AGPL-3.0-or-later" def test_the_console_image_serves_a_build_not_a_development_server() -> None: text = (ROOT / "frontend" / "Dockerfile").read_text("utf-8") assert "npm run build" in text assert "nginx-unprivileged" in text assert "npm run dev" not in text def test_the_node_agent_compose_projection_accepts_an_exact_release_image() -> None: text = (ROOT / "docker-compose.node-agent.yml").read_text("utf-8") assert "MODELFORGE_NODE_AGENT_IMAGE" in text assert "image:" in text assert "build:" in text, "the optional local source-build workflow must remain available" assert "latest" not in next( line for line in text.splitlines() if line.strip().startswith("image:") ) def test_the_production_control_plane_never_falls_back_to_latest() -> None: text = (ROOT / "docker-compose.production.yml").read_text("utf-8") image_lines = [line for line in text.splitlines() if line.strip().startswith("image:")] assert len(image_lines) == 3 assert all("latest" not in line for line in image_lines) assert all("MODELFORGE_VERSION:?" in line for line in image_lines) assert text.count("build:") == 2, "the explicit local source-build workflow must remain" def test_the_release_builder_never_mutates_a_floating_latest_tag() -> None: source = (ROOT / "scripts" / "release_build.py").read_text("utf-8") assert 'f"{image}:latest"' not in source def test_upgrade_runbook_starts_prebuilt_release_images() -> None: runbook = (ROOT / "docs" / "UPGRADE.md").read_text("utf-8") start_step = runbook.split("## 5. Start the new version", 1)[1].split("## 6. Verify", 1)[0] version = (ROOT / "VERSION").read_text("utf-8").strip() assert f"MODELFORGE_VERSION={version}" in start_step assert f"MODELFORGE_API_IMAGE=modelforge-api:{version}" in start_step assert f"MODELFORGE_WEB_IMAGE=modelforge-web:{version}" in start_step assert "up -d --no-build api web" in start_step assert "up -d --build api web" not in start_step # --------------------------------------------------------------------------- release build def test_the_release_package_carries_no_secret_material() -> None: """Whatever else changes, the tarball must never contain a credential or a database dump.""" from importlib.util import module_from_spec, spec_from_file_location spec = spec_from_file_location("release_build", ROOT / "scripts" / "release_build.py") assert spec and spec.loader module = module_from_spec(spec) sys.modules["release_build"] = module spec.loader.exec_module(module) # Match on what a file *is*, not on what its name mentions: docker-compose.backup.yml is a # deployment manifest for taking backups, not a backup, and a substring rule that cannot tell # those apart is a rule nobody will trust the next time it fires. secret_suffixes = {".key", ".pem", ".p12", ".pfx", ".crt", ".sql", ".dump", ".tar", ".gz"} secret_names = {".env", "secrets.yaml", "secrets.yml", "secrets.json", "credentials.json"} for path in module.ARTIFACT_PATHS: name = PurePosixPath(path).name.lower() assert name not in secret_names, f"{path} is secret material and must not be packaged" assert PurePosixPath(name).suffix not in secret_suffixes, ( f"{path} has a {PurePosixPath(name).suffix} extension and must not be packaged" ) assert ".env" not in module.ARTIFACT_PATHS assert ".env.example" in module.ARTIFACT_PATHS def test_the_release_package_carries_no_model_weights() -> None: from importlib.util import module_from_spec, spec_from_file_location spec = spec_from_file_location("release_build2", ROOT / "scripts" / "release_build.py") assert spec and spec.loader module = module_from_spec(spec) spec.loader.exec_module(module) for path in module.ARTIFACT_PATHS: assert "artifact" not in path.lower() assert "model-registry" not in path.lower() def test_the_release_package_includes_what_an_operator_needs_to_install() -> None: from importlib.util import module_from_spec, spec_from_file_location spec = spec_from_file_location("release_build3", ROOT / "scripts" / "release_build.py") assert spec and spec.loader module = module_from_spec(spec) spec.loader.exec_module(module) paths = set(module.ARTIFACT_PATHS) for required in ( "docker-compose.yml", "docker-compose.production.yml", ".env.example", "VERSION", "docs", "config", "scripts/bootstrap.py", "scripts/preflight.py", ): assert required in paths, f"a release without {required} cannot be installed from" def test_the_version_file_matches_the_product_version() -> None: assert (ROOT / "VERSION").read_text("utf-8").strip() == PRODUCT_VERSION # --------------------------------------------------------------------------- migration targeting def test_the_migration_environment_honours_an_explicitly_supplied_url() -> None: """A migration must run where the caller aimed it, not where the settings point. env.py used to overwrite `sqlalchemy.url` with the settings default unconditionally, so both `-x db_url=...` and a programmatic `set_main_option` were silently discarded. A bootstrap or upgrade rehearsal aimed at an isolated copy would have migrated the deployment's own database while reporting success against the copy. """ text = (ROOT / "backend" / "alembic" / "env.py").read_text("utf-8") assert "get_x_argument" in text, "-x db_url must be honoured" assert 'config.get_main_option("sqlalchemy.url", None)' in text, ( "a programmatically supplied URL must be honoured" ) unconditional = 'config.set_main_option("sqlalchemy.url", get_settings().database_url)' assert unconditional not in text, ( "the settings URL must be a fallback, never an unconditional override" ) def test_the_migration_configuration_hardcodes_no_database_url() -> None: """A URL baked into alembic.ini is a URL an operator can migrate the wrong database with.""" for line in (ROOT / "backend" / "alembic.ini").read_text("utf-8").splitlines(): if line.strip().startswith("sqlalchemy.url"): _, _, value = line.partition("=") assert not value.strip(), f"alembic.ini pins a database URL: {value.strip()!r}" # --------------------------------------------------------------------------- clean checkout def _tracked_files() -> set[str]: completed = subprocess.run( ["git", "ls-files"], # noqa: S607 - git resolves from PATH, as it must capture_output=True, text=True, encoding="utf-8", errors="replace", cwd=ROOT, check=True, ) return set(completed.stdout.split()) def test_every_manifest_the_control_plane_needs_at_startup_is_tracked() -> None: """A clean clone must be able to start. Ours could not. `.gitignore` carried a bare `models/` rule intended for model weights. It also matched `config/models/`, so the candidate registry manifest was silently excluded from the repository, and the API failed at startup with FileNotFoundError whenever registry seeding was enabled. The fresh-install rehearsal only passed because it mounted the developer's untracked copy — which is exactly the developer-state dependency a release is supposed to rule out. """ tracked = _tracked_files() required = [ "config/models/initial-candidates.yaml", "config/policies/defaults.yaml", ] missing = [path for path in required if path not in tracked] assert not missing, f"a clean checkout would be missing: {missing}" @pytest.mark.parametrize( "directory", ["config", "backend/src", "backend/tests", "node-agent/src", "runtime-worker/src", "docs"], ) def test_no_source_file_is_excluded_from_the_repository(directory: str) -> None: """Whatever these directories grow, none of it may be invisible to a clean clone. Two over-broad ignore rules each swallowed something that was reported as delivered. `models/` hid the candidate registry manifest the control plane needs at startup, so a clean clone could not start. `*credential*` hid backend/tests/test_credential_security_m16.py — the whole M16 credential security suite, 35 tests — and its documentation. Both existed on the machine that wrote them and nowhere else. """ tracked = _tracked_files() suffixes = {".py", ".ts", ".tsx", ".md", ".yaml", ".yml", ".json", ".inc", ".template"} on_disk = { str(path.relative_to(ROOT)).replace("\\", "/") for path in (ROOT / directory).rglob("*") if path.is_file() and path.suffix in suffixes and not any( part in {"__pycache__", "node_modules", ".pytest_cache", ".ruff_cache", ".mypy_cache"} or part.endswith(".egg-info") for part in path.parts ) } untracked = sorted(on_disk - tracked) assert not untracked, ( f"these files exist locally but not in the repository, so a clean clone would not have " f"them: {untracked}" ) # --------------------------------------------------------------------------- readiness cost def test_the_manifest_registry_reads_each_manifest_once() -> None: """Readiness must not re-validate the whole manifest set on every probe. M16 measured a p50 of 638 ms on /api/v1/health/ready under load and reported it as unexplained. The cause was here: one readiness pass performed 77 YAML reads across 14 files — one file ten times — because every accessor re-read from disk and projects() called capabilities() inside a nested loop. Manifests are read-only configuration that cannot change without restarting the process, so they are parsed once. """ from modelforge_api.services.manifest_registry import ManifestRegistry reads: list[str] = [] original = ManifestRegistry._read_yaml def counting(path: Path) -> dict[str, object]: reads.append(str(path)) return original(path) registry = ManifestRegistry(ROOT / "config") ManifestRegistry._read_yaml = staticmethod(counting) # type: ignore[method-assign] try: for _ in range(3): registry.capabilities() registry.projects() registry.candidates() registry.benchmarks() registry.policies() finally: ManifestRegistry._read_yaml = staticmethod(original) # type: ignore[method-assign] assert len(reads) == len(set(reads)), ( f"a manifest was read more than once: {sorted(reads)}" ) # --------------------------------------------------------------------------- content security def _csp_template() -> str: return (ROOT / "frontend" / "security-headers.inc.template").read_text("utf-8") def test_the_console_declares_a_content_security_policy() -> None: """M16 shipped six security headers and no CSP, and said so. v1 closes that.""" assert "Content-Security-Policy" in _csp_template() @pytest.mark.parametrize( "directive", [ "default-src 'none'", "script-src 'self'", "base-uri 'none'", "frame-ancestors 'none'", "object-src 'none'", ], ) def test_the_policy_is_restrictive_where_it_can_be(directive: str) -> None: assert directive in _csp_template() def test_the_policy_never_allows_inline_or_evaluated_script() -> None: """The two directives that would make the rest of the policy decorative.""" policy = _csp_template() script_directive = policy.split("script-src", 1)[1].split(";", 1)[0] assert "unsafe-inline" not in script_directive assert "unsafe-eval" not in script_directive def test_inline_style_is_permitted_only_as_an_attribute() -> None: """The console sets five dynamic widths through style attributes and nothing else. `style-src-attr 'unsafe-inline'` allows exactly those while still blocking an injected `