681 lines
28 KiB
Python
681 lines
28 KiB
Python
"""Packaging and configuration guarantees for the v1 release.
|
|
|
|
The release artefact is what an operator receives. These tests hold it to the things that are only
|
|
noticeable once it is too late: a secret that shipped, a production overlay that quietly accepts a
|
|
development default, an image without the identity to trace it back to a commit.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import re
|
|
import subprocess
|
|
import sys
|
|
import tomllib
|
|
from pathlib import Path, PurePosixPath
|
|
|
|
import pytest
|
|
|
|
from modelforge_api.domain.configuration_reference import (
|
|
DEPLOYMENT_DOCS,
|
|
SETTING_DOCS,
|
|
Sensitivity,
|
|
)
|
|
from modelforge_api.domain.release import PRODUCT_VERSION
|
|
from modelforge_api.settings import Settings
|
|
|
|
ROOT = Path(__file__).resolve().parents[2]
|
|
DOCKERFILES = (
|
|
"backend/Dockerfile",
|
|
"frontend/Dockerfile",
|
|
"node-agent/Dockerfile",
|
|
"runtime-worker/Dockerfile",
|
|
)
|
|
|
|
|
|
# --------------------------------------------------------------------------- configuration
|
|
|
|
|
|
def test_every_setting_is_documented() -> None:
|
|
"""A setting added without documentation fails the build rather than shipping quietly."""
|
|
|
|
undocumented = sorted(set(Settings.model_fields) - set(SETTING_DOCS))
|
|
assert not undocumented, f"undocumented settings: {undocumented}"
|
|
|
|
|
|
def test_the_reference_documents_no_setting_that_no_longer_exists() -> None:
|
|
stale = sorted(set(SETTING_DOCS) - set(Settings.model_fields))
|
|
assert not stale, f"documented settings that no longer exist: {stale}"
|
|
|
|
|
|
def test_the_generated_configuration_files_are_current() -> None:
|
|
"""`.env.example` and the configuration reference are generated, never hand-edited."""
|
|
|
|
completed = subprocess.run( # noqa: S603 - fixed argv, no shell
|
|
[sys.executable, str(ROOT / "scripts" / "generate_configuration_docs.py"), "--check"],
|
|
capture_output=True,
|
|
text=True,
|
|
encoding="utf-8",
|
|
errors="replace",
|
|
cwd=ROOT,
|
|
check=False,
|
|
)
|
|
assert completed.returncode == 0, (
|
|
"configuration documentation is stale; run "
|
|
f"scripts/generate_configuration_docs.py\n{completed.stdout}{completed.stderr}"
|
|
)
|
|
|
|
|
|
def test_the_example_configuration_carries_no_secret_values() -> None:
|
|
"""The example is committed, so a real value in it would be published with the release."""
|
|
|
|
text = (ROOT / ".env.example").read_text("utf-8")
|
|
secret_names = [
|
|
f"MODELFORGE_{name.upper()}"
|
|
for name, doc in SETTING_DOCS.items()
|
|
if doc.sensitivity is Sensitivity.SECRET
|
|
]
|
|
for name in secret_names:
|
|
for line in text.splitlines():
|
|
if line.startswith(f"{name}="):
|
|
assert line == f"{name}=", f"{name} carries a value in .env.example"
|
|
|
|
|
|
def test_every_required_production_setting_appears_in_the_example() -> None:
|
|
text = (ROOT / ".env.example").read_text("utf-8")
|
|
for name, doc in SETTING_DOCS.items():
|
|
if doc.required_in_production:
|
|
assert f"MODELFORGE_{name.upper()}=" in text, f"{name} is required but not offered"
|
|
|
|
|
|
def test_deployment_variables_are_documented_too() -> None:
|
|
"""Variables Compose and the agents read are still variables an operator has to set."""
|
|
|
|
text = (ROOT / ".env.example").read_text("utf-8")
|
|
for name in DEPLOYMENT_DOCS:
|
|
assert f"{name}=" in text, f"{name} is undocumented in .env.example"
|
|
|
|
|
|
# --------------------------------------------------------------------------- production overlay
|
|
|
|
|
|
def test_the_production_overlay_exists_and_sets_the_production_profile() -> None:
|
|
"""Without MODELFORGE_ENV=production every fail-closed startup rule stays switched off."""
|
|
|
|
text = (ROOT / "docker-compose.production.yml").read_text("utf-8")
|
|
assert "MODELFORGE_ENV: production" in text
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"variable",
|
|
[
|
|
"MODELFORGE_POSTGRES_ADMIN_PASSWORD",
|
|
"MODELFORGE_MIGRATION_DB_PASSWORD",
|
|
"MODELFORGE_RUNTIME_DB_PASSWORD",
|
|
"MODELFORGE_MIGRATION_DATABASE_URL",
|
|
"MODELFORGE_RUNTIME_DATABASE_URL",
|
|
"MODELFORGE_OPERATOR_API_KEY",
|
|
"MODELFORGE_BACKUP_ENCRYPTION_KEY",
|
|
"MODELFORGE_CORS_ORIGINS",
|
|
],
|
|
)
|
|
def test_the_production_overlay_refuses_to_render_without_its_secrets(variable: str) -> None:
|
|
"""`${VAR:?message}` makes Compose fail before a single container starts."""
|
|
|
|
text = (ROOT / "docker-compose.production.yml").read_text("utf-8")
|
|
assert re.search(rf"\$\{{{variable}:\?[^}}]+\}}", text), (
|
|
f"{variable} must use the ${{VAR:?message}} form so a missing value fails the render"
|
|
)
|
|
|
|
|
|
def test_the_production_overlay_never_permits_remote_code() -> None:
|
|
text = (ROOT / "docker-compose.production.yml").read_text("utf-8")
|
|
assert 'MODELFORGE_ALLOW_REMOTE_CODE: "false"' in text
|
|
|
|
|
|
def test_the_base_compose_file_is_not_mistakable_for_production() -> None:
|
|
"""The base file is development-only; role secrets are still required and never embedded."""
|
|
|
|
text = (ROOT / "docker-compose.yml").read_text("utf-8")
|
|
assert "MODELFORGE_ENV: production" not in text
|
|
|
|
|
|
# --------------------------------------------------------------------------- images
|
|
|
|
|
|
@pytest.mark.parametrize("dockerfile", DOCKERFILES)
|
|
def test_every_image_carries_oci_identity_labels(dockerfile: str) -> None:
|
|
text = (ROOT / dockerfile).read_text("utf-8")
|
|
for label in (
|
|
"org.opencontainers.image.version",
|
|
"org.opencontainers.image.revision",
|
|
"org.opencontainers.image.created",
|
|
"org.opencontainers.image.source",
|
|
):
|
|
assert label in text, f"{dockerfile} does not declare {label}"
|
|
|
|
|
|
@pytest.mark.parametrize("dockerfile", DOCKERFILES)
|
|
def test_every_image_accepts_build_identity_arguments(dockerfile: str) -> None:
|
|
text = (ROOT / dockerfile).read_text("utf-8")
|
|
for argument in ("MODELFORGE_VERSION", "MODELFORGE_COMMIT", "MODELFORGE_BUILT_AT"):
|
|
assert f"ARG {argument}" in text, f"{dockerfile} does not accept {argument}"
|
|
|
|
|
|
@pytest.mark.parametrize("dockerfile", DOCKERFILES)
|
|
def test_every_image_declares_the_product_license(dockerfile: str) -> None:
|
|
text = (ROOT / dockerfile).read_text("utf-8")
|
|
assert 'org.opencontainers.image.licenses="AGPL-3.0-or-later"' in text
|
|
assert 'org.opencontainers.image.licenses="Proprietary"' not in text
|
|
|
|
|
|
def test_every_component_declares_the_product_license() -> None:
|
|
for relative_path in (
|
|
"backend/pyproject.toml",
|
|
"node-agent/pyproject.toml",
|
|
"runtime-worker/pyproject.toml",
|
|
):
|
|
metadata = tomllib.loads((ROOT / relative_path).read_text("utf-8"))
|
|
assert metadata["project"]["license"] == "AGPL-3.0-or-later"
|
|
|
|
package = json.loads((ROOT / "frontend/package.json").read_text("utf-8"))
|
|
lock = json.loads((ROOT / "frontend/package-lock.json").read_text("utf-8"))
|
|
assert package["license"] == "AGPL-3.0-or-later"
|
|
assert lock["packages"][""]["license"] == "AGPL-3.0-or-later"
|
|
|
|
|
|
def test_the_console_image_serves_a_build_not_a_development_server() -> None:
|
|
text = (ROOT / "frontend" / "Dockerfile").read_text("utf-8")
|
|
assert "npm run build" in text
|
|
assert "nginx-unprivileged" in text
|
|
assert "npm run dev" not in text
|
|
|
|
|
|
def test_the_node_agent_compose_projection_accepts_an_exact_release_image() -> None:
|
|
text = (ROOT / "docker-compose.node-agent.yml").read_text("utf-8")
|
|
assert "MODELFORGE_NODE_AGENT_IMAGE" in text
|
|
assert "image:" in text
|
|
assert "build:" in text, "the optional local source-build workflow must remain available"
|
|
assert "latest" not in next(
|
|
line for line in text.splitlines() if line.strip().startswith("image:")
|
|
)
|
|
|
|
|
|
def test_the_production_control_plane_never_falls_back_to_latest() -> None:
|
|
text = (ROOT / "docker-compose.production.yml").read_text("utf-8")
|
|
image_lines = [line for line in text.splitlines() if line.strip().startswith("image:")]
|
|
assert len(image_lines) == 3
|
|
assert all("latest" not in line for line in image_lines)
|
|
assert all("MODELFORGE_VERSION:?" in line for line in image_lines)
|
|
assert text.count("build:") == 2, "the explicit local source-build workflow must remain"
|
|
|
|
|
|
def test_the_release_builder_never_mutates_a_floating_latest_tag() -> None:
|
|
source = (ROOT / "scripts" / "release_build.py").read_text("utf-8")
|
|
assert 'f"{image}:latest"' not in source
|
|
|
|
|
|
def test_upgrade_runbook_starts_prebuilt_release_images() -> None:
|
|
runbook = (ROOT / "docs" / "UPGRADE.md").read_text("utf-8")
|
|
start_step = runbook.split("## 5. Start the new version", 1)[1].split("## 6. Verify", 1)[0]
|
|
version = (ROOT / "VERSION").read_text("utf-8").strip()
|
|
assert f"MODELFORGE_VERSION={version}" in start_step
|
|
assert f"MODELFORGE_API_IMAGE=modelforge-api:{version}" in start_step
|
|
assert f"MODELFORGE_WEB_IMAGE=modelforge-web:{version}" in start_step
|
|
assert "up -d --no-build api web" in start_step
|
|
assert "up -d --build api web" not in start_step
|
|
|
|
|
|
# --------------------------------------------------------------------------- release build
|
|
|
|
|
|
def test_the_release_package_carries_no_secret_material() -> None:
|
|
"""Whatever else changes, the tarball must never contain a credential or a database dump."""
|
|
|
|
from importlib.util import module_from_spec, spec_from_file_location
|
|
|
|
spec = spec_from_file_location("release_build", ROOT / "scripts" / "release_build.py")
|
|
assert spec and spec.loader
|
|
module = module_from_spec(spec)
|
|
sys.modules["release_build"] = module
|
|
spec.loader.exec_module(module)
|
|
|
|
# Match on what a file *is*, not on what its name mentions: docker-compose.backup.yml is a
|
|
# deployment manifest for taking backups, not a backup, and a substring rule that cannot tell
|
|
# those apart is a rule nobody will trust the next time it fires.
|
|
secret_suffixes = {".key", ".pem", ".p12", ".pfx", ".crt", ".sql", ".dump", ".tar", ".gz"}
|
|
secret_names = {".env", "secrets.yaml", "secrets.yml", "secrets.json", "credentials.json"}
|
|
for path in module.ARTIFACT_PATHS:
|
|
name = PurePosixPath(path).name.lower()
|
|
assert name not in secret_names, f"{path} is secret material and must not be packaged"
|
|
assert PurePosixPath(name).suffix not in secret_suffixes, (
|
|
f"{path} has a {PurePosixPath(name).suffix} extension and must not be packaged"
|
|
)
|
|
assert ".env" not in module.ARTIFACT_PATHS
|
|
assert ".env.example" in module.ARTIFACT_PATHS
|
|
|
|
|
|
def test_the_release_package_carries_no_model_weights() -> None:
|
|
from importlib.util import module_from_spec, spec_from_file_location
|
|
|
|
spec = spec_from_file_location("release_build2", ROOT / "scripts" / "release_build.py")
|
|
assert spec and spec.loader
|
|
module = module_from_spec(spec)
|
|
spec.loader.exec_module(module)
|
|
for path in module.ARTIFACT_PATHS:
|
|
assert "artifact" not in path.lower()
|
|
assert "model-registry" not in path.lower()
|
|
|
|
|
|
def test_the_release_package_includes_what_an_operator_needs_to_install() -> None:
|
|
from importlib.util import module_from_spec, spec_from_file_location
|
|
|
|
spec = spec_from_file_location("release_build3", ROOT / "scripts" / "release_build.py")
|
|
assert spec and spec.loader
|
|
module = module_from_spec(spec)
|
|
spec.loader.exec_module(module)
|
|
paths = set(module.ARTIFACT_PATHS)
|
|
for required in (
|
|
"docker-compose.yml",
|
|
"docker-compose.production.yml",
|
|
".env.example",
|
|
"VERSION",
|
|
"docs",
|
|
"config",
|
|
"scripts/bootstrap.py",
|
|
"scripts/preflight.py",
|
|
):
|
|
assert required in paths, f"a release without {required} cannot be installed from"
|
|
|
|
|
|
def test_the_version_file_matches_the_product_version() -> None:
|
|
assert (ROOT / "VERSION").read_text("utf-8").strip() == PRODUCT_VERSION
|
|
|
|
|
|
# --------------------------------------------------------------------------- migration targeting
|
|
|
|
|
|
def test_the_migration_environment_honours_an_explicitly_supplied_url() -> None:
|
|
"""A migration must run where the caller aimed it, not where the settings point.
|
|
|
|
env.py used to overwrite `sqlalchemy.url` with the settings default unconditionally, so both
|
|
`-x db_url=...` and a programmatic `set_main_option` were silently discarded. A bootstrap or
|
|
upgrade rehearsal aimed at an isolated copy would have migrated the deployment's own database
|
|
while reporting success against the copy.
|
|
"""
|
|
|
|
text = (ROOT / "backend" / "alembic" / "env.py").read_text("utf-8")
|
|
assert "get_x_argument" in text, "-x db_url must be honoured"
|
|
assert 'config.get_main_option("sqlalchemy.url", None)' in text, (
|
|
"a programmatically supplied URL must be honoured"
|
|
)
|
|
unconditional = 'config.set_main_option("sqlalchemy.url", get_settings().database_url)'
|
|
assert unconditional not in text, (
|
|
"the settings URL must be a fallback, never an unconditional override"
|
|
)
|
|
|
|
|
|
def test_the_migration_configuration_hardcodes_no_database_url() -> None:
|
|
"""A URL baked into alembic.ini is a URL an operator can migrate the wrong database with."""
|
|
|
|
for line in (ROOT / "backend" / "alembic.ini").read_text("utf-8").splitlines():
|
|
if line.strip().startswith("sqlalchemy.url"):
|
|
_, _, value = line.partition("=")
|
|
assert not value.strip(), f"alembic.ini pins a database URL: {value.strip()!r}"
|
|
|
|
|
|
# --------------------------------------------------------------------------- clean checkout
|
|
|
|
|
|
def _tracked_files() -> set[str]:
|
|
completed = subprocess.run(
|
|
["git", "ls-files"], # noqa: S607 - git resolves from PATH, as it must
|
|
capture_output=True,
|
|
text=True,
|
|
encoding="utf-8",
|
|
errors="replace",
|
|
cwd=ROOT,
|
|
check=True,
|
|
)
|
|
return set(completed.stdout.split())
|
|
|
|
|
|
def test_every_manifest_the_control_plane_needs_at_startup_is_tracked() -> None:
|
|
"""A clean clone must be able to start. Ours could not.
|
|
|
|
`.gitignore` carried a bare `models/` rule intended for model weights. It also matched
|
|
`config/models/`, so the candidate registry manifest was silently excluded from the repository,
|
|
and the API failed at startup with FileNotFoundError whenever registry seeding was enabled. The
|
|
fresh-install rehearsal only passed because it mounted the developer's untracked copy — which is
|
|
exactly the developer-state dependency a release is supposed to rule out.
|
|
"""
|
|
|
|
tracked = _tracked_files()
|
|
required = [
|
|
"config/models/initial-candidates.yaml",
|
|
"config/policies/defaults.yaml",
|
|
]
|
|
missing = [path for path in required if path not in tracked]
|
|
assert not missing, f"a clean checkout would be missing: {missing}"
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"directory",
|
|
["config", "backend/src", "backend/tests", "node-agent/src", "runtime-worker/src", "docs"],
|
|
)
|
|
def test_no_source_file_is_excluded_from_the_repository(directory: str) -> None:
|
|
"""Whatever these directories grow, none of it may be invisible to a clean clone.
|
|
|
|
Two over-broad ignore rules each swallowed something that was reported as delivered. `models/`
|
|
hid the candidate registry manifest the control plane needs at startup, so a clean clone could
|
|
not start. `*credential*` hid backend/tests/test_credential_security_m16.py — the whole M16
|
|
credential security suite, 35 tests — and its documentation. Both existed on the machine that
|
|
wrote them and nowhere else.
|
|
"""
|
|
|
|
tracked = _tracked_files()
|
|
suffixes = {".py", ".ts", ".tsx", ".md", ".yaml", ".yml", ".json", ".inc", ".template"}
|
|
on_disk = {
|
|
str(path.relative_to(ROOT)).replace("\\", "/")
|
|
for path in (ROOT / directory).rglob("*")
|
|
if path.is_file()
|
|
and path.suffix in suffixes
|
|
and not any(
|
|
part in {"__pycache__", "node_modules", ".pytest_cache", ".ruff_cache", ".mypy_cache"}
|
|
or part.endswith(".egg-info")
|
|
for part in path.parts
|
|
)
|
|
}
|
|
untracked = sorted(on_disk - tracked)
|
|
assert not untracked, (
|
|
f"these files exist locally but not in the repository, so a clean clone would not have "
|
|
f"them: {untracked}"
|
|
)
|
|
|
|
|
|
# --------------------------------------------------------------------------- readiness cost
|
|
|
|
|
|
def test_the_manifest_registry_reads_each_manifest_once() -> None:
|
|
"""Readiness must not re-validate the whole manifest set on every probe.
|
|
|
|
M16 measured a p50 of 638 ms on /api/v1/health/ready under load and reported it as unexplained.
|
|
The cause was here: one readiness pass performed 77 YAML reads across 14 files — one file ten
|
|
times — because every accessor re-read from disk and projects() called capabilities() inside a
|
|
nested loop. Manifests are read-only configuration that cannot change without restarting the
|
|
process, so they are parsed once.
|
|
"""
|
|
|
|
from modelforge_api.services.manifest_registry import ManifestRegistry
|
|
|
|
reads: list[str] = []
|
|
original = ManifestRegistry._read_yaml
|
|
|
|
def counting(path: Path) -> dict[str, object]:
|
|
reads.append(str(path))
|
|
return original(path)
|
|
|
|
registry = ManifestRegistry(ROOT / "config")
|
|
ManifestRegistry._read_yaml = staticmethod(counting) # type: ignore[method-assign]
|
|
try:
|
|
for _ in range(3):
|
|
registry.capabilities()
|
|
registry.projects()
|
|
registry.candidates()
|
|
registry.benchmarks()
|
|
registry.policies()
|
|
finally:
|
|
ManifestRegistry._read_yaml = staticmethod(original) # type: ignore[method-assign]
|
|
|
|
assert len(reads) == len(set(reads)), (
|
|
f"a manifest was read more than once: {sorted(reads)}"
|
|
)
|
|
|
|
|
|
# --------------------------------------------------------------------------- content security
|
|
|
|
|
|
def _csp_template() -> str:
|
|
return (ROOT / "frontend" / "security-headers.inc.template").read_text("utf-8")
|
|
|
|
|
|
def test_the_console_declares_a_content_security_policy() -> None:
|
|
"""M16 shipped six security headers and no CSP, and said so. v1 closes that."""
|
|
|
|
assert "Content-Security-Policy" in _csp_template()
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"directive",
|
|
[
|
|
"default-src 'none'",
|
|
"script-src 'self'",
|
|
"base-uri 'none'",
|
|
"frame-ancestors 'none'",
|
|
"object-src 'none'",
|
|
],
|
|
)
|
|
def test_the_policy_is_restrictive_where_it_can_be(directive: str) -> None:
|
|
assert directive in _csp_template()
|
|
|
|
|
|
def test_the_policy_never_allows_inline_or_evaluated_script() -> None:
|
|
"""The two directives that would make the rest of the policy decorative."""
|
|
|
|
policy = _csp_template()
|
|
script_directive = policy.split("script-src", 1)[1].split(";", 1)[0]
|
|
assert "unsafe-inline" not in script_directive
|
|
assert "unsafe-eval" not in script_directive
|
|
|
|
|
|
def test_inline_style_is_permitted_only_as_an_attribute() -> None:
|
|
"""The console sets five dynamic widths through style attributes and nothing else.
|
|
|
|
`style-src-attr 'unsafe-inline'` allows exactly those while still blocking an injected
|
|
`<style>` element, which a blanket `style-src 'unsafe-inline'` would not.
|
|
"""
|
|
|
|
policy = _csp_template()
|
|
style_directive = policy.split("style-src ", 1)[1].split(";", 1)[0]
|
|
assert "unsafe-inline" not in style_directive
|
|
assert "style-src-attr 'unsafe-inline'" in policy
|
|
|
|
|
|
def test_the_built_console_contains_nothing_the_policy_forbids() -> None:
|
|
"""A policy the application violates is a policy someone will switch off.
|
|
|
|
Checked against the built bundle rather than asserted: no inline <script>, no inline <style>,
|
|
and no javascript: URL. The favicon is a data: URI, which img-src permits.
|
|
"""
|
|
|
|
index = ROOT / "frontend" / "dist" / "index.html"
|
|
if not index.is_file():
|
|
pytest.skip("the console has not been built in this working tree")
|
|
html = index.read_text("utf-8")
|
|
assert not re.search(r"<script(?![^>]*\ssrc=)[^>]*>", html), "an inline <script> would be blocked"
|
|
assert "<style" not in html, "an inline <style> element would be blocked"
|
|
assert "javascript:" not in html
|
|
for match in re.findall(r'src="([^"]+)"|href="([^"]+)"', html):
|
|
value = match[0] or match[1]
|
|
assert value.startswith(("/", "./", "data:")), (
|
|
f"{value} is an external reference the policy does not allow"
|
|
)
|
|
|
|
|
|
# --------------------------------------------------------------------------- documentation
|
|
|
|
|
|
def test_the_operator_documentation_a_release_promises_is_present() -> None:
|
|
"""An operator should not need a milestone report to install or run ModelForge."""
|
|
|
|
required = [
|
|
"README.md",
|
|
"CHANGELOG.md",
|
|
"docs/INSTALLATION.md",
|
|
"docs/FIRST_RUN.md",
|
|
"docs/UPGRADE.md",
|
|
"docs/CONFIGURATION.md",
|
|
"docs/COMPATIBILITY.md",
|
|
"docs/NODE_AGENT.md",
|
|
"docs/UNRAID_DEPLOYMENT.md",
|
|
"docs/CAPABILITIES.md",
|
|
"docs/PROJECT_INTEGRATION.md",
|
|
"docs/OPERATIONS.md",
|
|
"docs/TROUBLESHOOTING.md",
|
|
"docs/SECURITY.md",
|
|
f"docs/RELEASE_NOTES_v{PRODUCT_VERSION}.md",
|
|
]
|
|
missing = [path for path in required if not (ROOT / path).is_file()]
|
|
assert not missing, f"missing operator documentation: {missing}"
|
|
|
|
|
|
def test_no_documentation_link_is_broken() -> None:
|
|
"""Documentation that points at a file which is not there is worse than none."""
|
|
|
|
broken: list[str] = []
|
|
documents = list(ROOT.glob("*.md")) + list((ROOT / "docs").rglob("*.md"))
|
|
for document in documents:
|
|
for target in re.findall(r"\]\(([^)#]+?)(?:#[^)]*)?\)", document.read_text("utf-8")):
|
|
if target.startswith(("http://", "https://", "mailto:")):
|
|
continue
|
|
if not (document.parent / target).resolve().exists():
|
|
broken.append(f"{document.relative_to(ROOT).as_posix()} -> {target}")
|
|
assert not broken, f"broken documentation links: {broken}"
|
|
|
|
|
|
def test_the_release_notes_name_this_version() -> None:
|
|
notes = (ROOT / "docs" / f"RELEASE_NOTES_v{PRODUCT_VERSION}.md").read_text("utf-8")
|
|
assert PRODUCT_VERSION in notes
|
|
assert "Known limitations" in notes, "a release that lists no limitations has not looked"
|
|
|
|
|
|
def test_the_release_archive_is_built_reproducibly() -> None:
|
|
"""Two builds of the same tree must produce the same bytes, or a published checksum means little.
|
|
|
|
Two things break this and both were found by building twice and comparing rather than by
|
|
reasoning about it: tar records uid, gid and mtime per entry, and gzip writes the current time
|
|
into its own header.
|
|
"""
|
|
|
|
source = (ROOT / "scripts" / "release_build.py").read_text("utf-8")
|
|
assert "mtime=0" in source, "the gzip header must not carry a build timestamp"
|
|
assert "info.mtime = 0" in source, "entry mtimes must be normalised"
|
|
assert 'info.uname = info.gname = "root"' in source, "entry ownership must be normalised"
|
|
assert "recursive=False" in source, "tar.add must not recurse and duplicate every entry"
|
|
|
|
|
|
def test_build_identity_is_never_overridden_at_deployment_time() -> None:
|
|
"""An image knows what it was built from; whoever runs `up` does not.
|
|
|
|
The Compose files used to pass MODELFORGE_BUILD_COMMIT and MODELFORGE_BUILD_TIMESTAMP as runtime
|
|
environment, which override the image's own values. The release candidate reported the commit
|
|
of whatever was checked out when it was started — not the commit its image was built from — and
|
|
a null build time, because the unset variable blanked what the image already carried.
|
|
"""
|
|
|
|
for name in ("docker-compose.yml", "docker-compose.production.yml"):
|
|
text = (ROOT / name).read_text("utf-8")
|
|
for variable in ("MODELFORGE_BUILD_COMMIT", "MODELFORGE_BUILD_TIMESTAMP"):
|
|
for line in text.splitlines():
|
|
stripped = line.strip()
|
|
if stripped.startswith(f"{variable}:"):
|
|
raise AssertionError(
|
|
f"{name} sets {variable} at runtime; build identity must come from the "
|
|
f"image's own build arguments"
|
|
)
|
|
# The build arguments themselves must still be accepted.
|
|
assert "MODELFORGE_COMMIT:" in text, f"{name} must pass the commit as a build argument"
|
|
|
|
|
|
# --------------------------------------------------------------------------- reproducible install
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"manifest",
|
|
["backend/pyproject.toml", "node-agent/pyproject.toml", "runtime-worker/pyproject.toml"],
|
|
)
|
|
def test_every_dependency_is_pinned_to_an_exact_version(manifest: str) -> None:
|
|
"""A release must resolve to the same versions on any day, on any machine.
|
|
|
|
It did not. `sqlalchemy>=2.0,<3` resolved to 2.0.35 in the development environment and 2.0.52
|
|
in a clean install — and 2.0.52 narrowed `Session.execute`'s return type, so the type check
|
|
passed locally and failed in CI on identical code. The gate depended on when you ran it.
|
|
"""
|
|
|
|
import tomllib
|
|
|
|
data = tomllib.loads((ROOT / manifest).read_text("utf-8"))
|
|
project = data["project"]
|
|
specs = list(project.get("dependencies", []))
|
|
for extra in project.get("optional-dependencies", {}).values():
|
|
specs.extend(extra)
|
|
|
|
unpinned = [
|
|
spec
|
|
for spec in specs
|
|
if "==" not in spec or any(operator in spec for operator in (">=", "<=", ">", "<", "~="))
|
|
]
|
|
assert not unpinned, f"{manifest} declares unpinned dependencies: {unpinned}"
|
|
|
|
|
|
def test_the_type_checker_version_is_pinned_everywhere_it_runs() -> None:
|
|
"""The gate must not change its answer because a tool released a new version overnight."""
|
|
|
|
import tomllib
|
|
|
|
seen: set[str] = set()
|
|
for manifest in (
|
|
"backend/pyproject.toml",
|
|
"node-agent/pyproject.toml",
|
|
"runtime-worker/pyproject.toml",
|
|
):
|
|
data = tomllib.loads((ROOT / manifest).read_text("utf-8"))
|
|
for extra in data["project"].get("optional-dependencies", {}).values():
|
|
for spec in extra:
|
|
if spec.startswith(("mypy", "ruff")):
|
|
seen.add(spec)
|
|
mypy_pins = {spec for spec in seen if spec.startswith("mypy")}
|
|
ruff_pins = {spec for spec in seen if spec.startswith("ruff")}
|
|
assert len(mypy_pins) == 1, f"components disagree on the mypy version: {sorted(mypy_pins)}"
|
|
assert len(ruff_pins) == 1, f"components disagree on the ruff version: {sorted(ruff_pins)}"
|
|
|
|
|
|
def test_the_example_configuration_uses_container_paths_not_host_paths() -> None:
|
|
"""The defaults are paths inside a Linux container, whatever platform generated the file.
|
|
|
|
`str(Path("/data/backups"))` yields `\\data\backups` on Windows, and the committed
|
|
.env.example shipped exactly that — telling an operator to point a Linux container at a Windows
|
|
path, and making the generated files differ by platform so the freshness check passed on one and
|
|
failed on the other.
|
|
"""
|
|
|
|
text = (ROOT / ".env.example").read_text("utf-8")
|
|
offenders = [
|
|
line
|
|
for line in text.splitlines()
|
|
if line.startswith("MODELFORGE_") and "\\" in line.partition("=")[2]
|
|
]
|
|
assert not offenders, f"host-style paths in .env.example: {offenders}"
|
|
|
|
|
|
def test_no_test_silently_requires_a_live_database() -> None:
|
|
"""A suite that passes only where a database happens to be running is not a suite.
|
|
|
|
test_api.py called an endpoint without overriding `get_session`, so it used the real engine and
|
|
quietly required PostgreSQL on localhost. It passed on the development machine and failed
|
|
everywhere else.
|
|
"""
|
|
|
|
source = (ROOT / "backend" / "tests" / "test_api.py").read_text("utf-8")
|
|
# Every test that reaches a session-backed route must install an override first.
|
|
uses_client = source.count("client.get(")
|
|
overrides = source.count("app.dependency_overrides[get_session]")
|
|
assert overrides >= 1, "test_api.py must override get_session rather than use the real engine"
|
|
# A DSN, not the word: the docstring above mentions PostgreSQL precisely because that is what
|
|
# went wrong, and a rule that cannot tell prose from a connection string is a rule that will be
|
|
# switched off the first time it fires wrongly.
|
|
assert not re.search(r"postgresql(\+\w+)?://", source), (
|
|
"no test may name a PostgreSQL DSN; the suite runs against SQLite"
|
|
)
|
|
assert uses_client > 0
|